1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/FoldingSet.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Triple.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/CodeGen/ISDOpcodes.h"
29#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineConstantPool.h"
31#include "llvm/CodeGen/MachineFrameInfo.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineMemOperand.h"
34#include "llvm/CodeGen/RuntimeLibcalls.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
38#include "llvm/CodeGen/TargetLowering.h"
39#include "llvm/CodeGen/TargetRegisterInfo.h"
40#include "llvm/CodeGen/TargetSubtargetInfo.h"
41#include "llvm/CodeGen/ValueTypes.h"
42#include "llvm/IR/Constant.h"
43#include "llvm/IR/Constants.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/IR/DebugLoc.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/DiagnosticInfo.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/Metadata.h"
52#include "llvm/IR/Type.h"
53#include "llvm/IR/Value.h"
54#include "llvm/Support/Casting.h"
55#include "llvm/Support/CodeGen.h"
56#include "llvm/Support/Compiler.h"
57#include "llvm/Support/Debug.h"
58#include "llvm/Support/ErrorHandling.h"
59#include "llvm/Support/KnownBits.h"
60#include "llvm/Support/MachineValueType.h"
61#include "llvm/Support/ManagedStatic.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Support/Mutex.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include "llvm/Transforms/Utils/CheriSetBounds.h"
68#include <algorithm>
69#include <cassert>
70#include <cstdint>
71#include <cstdlib>
72#include <limits>
73#include <set>
74#include <string>
75#include <utility>
76#include <vector>
77
78using namespace llvm;
79
80/// makeVTList - Return an instance of the SDVTList struct initialized with the
81/// specified members.
82static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
83 SDVTList Res = {VTs, NumVTs};
84 return Res;
85}
86
87// Default null implementations of the callbacks.
88void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
89void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
90void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
91
92void SelectionDAG::DAGNodeDeletedListener::anchor() {}
93
94#define DEBUG_TYPE "selectiondag"
95
96static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
97 cl::Hidden, cl::init(true),
98 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
99
100static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
101 cl::desc("Number limit for gluing ld/st of memcpy."),
102 cl::Hidden, cl::init(0));
103
104static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
105 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
106}
107
108//===----------------------------------------------------------------------===//
109// ConstantFPSDNode Class
110//===----------------------------------------------------------------------===//
111
112/// isExactlyValue - We don't rely on operator== working on double values, as
113/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
114/// As such, this method can be used to do an exact bit-for-bit comparison of
115/// two floating point values.
116bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
117 return getValueAPF().bitwiseIsEqual(V);
118}
119
120bool ConstantFPSDNode::isValueValidForType(EVT VT,
121 const APFloat& Val) {
122 assert(VT.isFloatingPoint() && "Can only convert between FP types");
123
124 // convert modifies in place, so make a copy.
125 APFloat Val2 = APFloat(Val);
126 bool losesInfo;
127 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
128 APFloat::rmNearestTiesToEven,
129 &losesInfo);
130 return !losesInfo;
131}
132
133//===----------------------------------------------------------------------===//
134// ISD Namespace
135//===----------------------------------------------------------------------===//
136
137bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
138 auto *BV = dyn_cast<BuildVectorSDNode>(N);
139 if (!BV)
140 return false;
141
142 APInt SplatUndef;
143 unsigned SplatBitSize;
144 bool HasUndefs;
145 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
146 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
147 EltSize) &&
148 EltSize == SplatBitSize;
149}
150
151// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
152// specializations of the more general isConstantSplatVector()?
153
154bool ISD::isBuildVectorAllOnes(const SDNode *N) {
155 // Look through a bit convert.
156 while (N->getOpcode() == ISD::BITCAST)
157 N = N->getOperand(0).getNode();
158
159 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
160
161 unsigned i = 0, e = N->getNumOperands();
162
163 // Skip over all of the undef values.
164 while (i != e && N->getOperand(i).isUndef())
165 ++i;
166
167 // Do not accept an all-undef vector.
168 if (i == e) return false;
169
170 // Do not accept build_vectors that aren't all constants or which have non-~0
171 // elements. We have to be a bit careful here, as the type of the constant
172 // may not be the same as the type of the vector elements due to type
173 // legalization (the elements are promoted to a legal type for the target and
174 // a vector of a type may be legal when the base element type is not).
175 // We only want to check enough bits to cover the vector elements, because
176 // we care if the resultant vector is all ones, not whether the individual
177 // constants are.
178 SDValue NotZero = N->getOperand(i);
179 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
180 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
181 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
182 return false;
183 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
184 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
185 return false;
186 } else
187 return false;
188
189 // Okay, we have at least one ~0 value, check to see if the rest match or are
190 // undefs. Even with the above element type twiddling, this should be OK, as
191 // the same type legalization should have applied to all the elements.
192 for (++i; i != e; ++i)
193 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
194 return false;
195 return true;
196}
197
198bool ISD::isBuildVectorAllZeros(const SDNode *N) {
199 // Look through a bit convert.
200 while (N->getOpcode() == ISD::BITCAST)
201 N = N->getOperand(0).getNode();
202
203 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
204
205 bool IsAllUndef = true;
206 for (const SDValue &Op : N->op_values()) {
207 if (Op.isUndef())
208 continue;
209 IsAllUndef = false;
210 // Do not accept build_vectors that aren't all constants or which have non-0
211 // elements. We have to be a bit careful here, as the type of the constant
212 // may not be the same as the type of the vector elements due to type
213 // legalization (the elements are promoted to a legal type for the target
214 // and a vector of a type may be legal when the base element type is not).
215 // We only want to check enough bits to cover the vector elements, because
216 // we care if the resultant vector is all zeros, not whether the individual
217 // constants are.
218 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
219 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
220 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
221 return false;
222 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
223 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
224 return false;
225 } else
226 return false;
227 }
228
229 // Do not accept an all-undef vector.
230 if (IsAllUndef)
231 return false;
232 return true;
233}
234
235bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
236 if (N->getOpcode() != ISD::BUILD_VECTOR)
237 return false;
238
239 for (const SDValue &Op : N->op_values()) {
240 if (Op.isUndef())
241 continue;
242 if (!isa<ConstantSDNode>(Op))
243 return false;
244 }
245 return true;
246}
247
248bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
249 if (N->getOpcode() != ISD::BUILD_VECTOR)
250 return false;
251
252 for (const SDValue &Op : N->op_values()) {
253 if (Op.isUndef())
254 continue;
255 if (!isa<ConstantFPSDNode>(Op))
256 return false;
257 }
258 return true;
259}
260
261bool ISD::allOperandsUndef(const SDNode *N) {
262 // Return false if the node has no operands.
263 // This is "logically inconsistent" with the definition of "all" but
264 // is probably the desired behavior.
265 if (N->getNumOperands() == 0)
266 return false;
267 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
268}
269
270bool ISD::matchUnaryPredicate(SDValue Op,
271 std::function<bool(ConstantSDNode *)> Match,
272 bool AllowUndefs) {
273 // FIXME: Add support for scalar UNDEF cases?
274 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
275 return Match(Cst);
276
277 // FIXME: Add support for vector UNDEF cases?
278 if (ISD::BUILD_VECTOR != Op.getOpcode())
279 return false;
280
281 EVT SVT = Op.getValueType().getScalarType();
282 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
283 if (AllowUndefs && Op.getOperand(i).isUndef()) {
284 if (!Match(nullptr))
285 return false;
286 continue;
287 }
288
289 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
290 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
291 return false;
292 }
293 return true;
294}
295
296bool ISD::matchBinaryPredicate(
297 SDValue LHS, SDValue RHS,
298 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
299 bool AllowUndefs) {
300 if (LHS.getValueType() != RHS.getValueType())
301 return false;
302
303 // TODO: Add support for scalar UNDEF cases?
304 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
305 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
306 return Match(LHSCst, RHSCst);
307
308 // TODO: Add support for vector UNDEF cases?
309 if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
310 ISD::BUILD_VECTOR != RHS.getOpcode())
311 return false;
312
313 EVT SVT = LHS.getValueType().getScalarType();
314 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
315 SDValue LHSOp = LHS.getOperand(i);
316 SDValue RHSOp = RHS.getOperand(i);
317 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
318 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
319 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
320 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
321 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
322 return false;
323 if (LHSOp.getValueType() != SVT ||
324 LHSOp.getValueType() != RHSOp.getValueType())
325 return false;
326 if (!Match(LHSCst, RHSCst))
327 return false;
328 }
329 return true;
330}
331
332ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
333 switch (ExtType) {
334 case ISD::EXTLOAD:
335 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
336 case ISD::SEXTLOAD:
337 return ISD::SIGN_EXTEND;
338 case ISD::ZEXTLOAD:
339 return ISD::ZERO_EXTEND;
340 default:
341 break;
342 }
343
344 llvm_unreachable("Invalid LoadExtType");
345}
346
347ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
348 // To perform this operation, we just need to swap the L and G bits of the
349 // operation.
350 unsigned OldL = (Operation >> 2) & 1;
351 unsigned OldG = (Operation >> 1) & 1;
352 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
353 (OldL << 1) | // New G bit
354 (OldG << 2)); // New L bit.
355}
356
357ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT VT) {
358 bool IsInteger = VT.isInteger();
359
360 unsigned Operation = Op;
361 if (IsInteger || VT.isFatPointer())
362 Operation ^= 7; // Flip L, G, E bits, but not U.
363 else
364 Operation ^= 15; // Flip all of the condition bits.
365
366 if (Operation > ISD::SETTRUE2)
367 Operation &= ~8; // Don't let N and U bits get set.
368
369 return ISD::CondCode(Operation);
370}
371
372/// For an integer comparison, return 1 if the comparison is a signed operation
373/// and 2 if the result is an unsigned comparison. Return zero if the operation
374/// does not depend on the sign of the input (setne and seteq).
375static int isSignedOp(ISD::CondCode Opcode) {
376 switch (Opcode) {
377 default: llvm_unreachable("Illegal integer setcc operation!");
378 case ISD::SETEQ:
379 case ISD::SETNE: return 0;
380 case ISD::SETLT:
381 case ISD::SETLE:
382 case ISD::SETGT:
383 case ISD::SETGE: return 1;
384 case ISD::SETULT:
385 case ISD::SETULE:
386 case ISD::SETUGT:
387 case ISD::SETUGE: return 2;
388 }
389}
390
391ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
392 EVT VT) {
393 // XXXAR: I don't think we can fold setcc or operations with fat pointers
394 if (VT.isFatPointer())
395 return ISD::SETCC_INVALID;
396 bool IsInteger = VT.isInteger();
397 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
398 // Cannot fold a signed integer setcc with an unsigned integer setcc.
399 return ISD::SETCC_INVALID;
400
401 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
402
403 // If the N and U bits get set, then the resultant comparison DOES suddenly
404 // care about orderedness, and it is true when ordered.
405 if (Op > ISD::SETTRUE2)
406 Op &= ~16; // Clear the U bit if the N bit is set.
407
408 // Canonicalize illegal integer setcc's.
409 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
410 Op = ISD::SETNE;
411
412 return ISD::CondCode(Op);
413}
414
415ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
416 EVT VT) {
417 // XXXAR: I don't think we can fold setcc and operations with fat pointers
418 if (VT.isFatPointer())
419 return ISD::SETCC_INVALID;
420 bool IsInteger = VT.isInteger();
421 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
422 // Cannot fold a signed setcc with an unsigned setcc.
423 return ISD::SETCC_INVALID;
424
425 // Combine all of the condition bits.
426 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
427
428 // Canonicalize illegal integer setcc's.
429 if (IsInteger) {
430 switch (Result) {
431 default: break;
432 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
433 case ISD::SETOEQ: // SETEQ & SETU[LG]E
434 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
435 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
436 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
437 }
438 }
439
440 return Result;
441}
442
443//===----------------------------------------------------------------------===//
444// SDNode Profile Support
445//===----------------------------------------------------------------------===//
446
447/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
448static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
449 ID.AddInteger(OpC);
450}
451
452/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
453/// solely with their pointer.
454static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
455 ID.AddPointer(VTList.VTs);
456}
457
458/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
459static void AddNodeIDOperands(FoldingSetNodeID &ID,
460 ArrayRef<SDValue> Ops) {
461 for (auto& Op : Ops) {
462 ID.AddPointer(Op.getNode());
463 ID.AddInteger(Op.getResNo());
464 }
465}
466
467/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
468static void AddNodeIDOperands(FoldingSetNodeID &ID,
469 ArrayRef<SDUse> Ops) {
470 for (auto& Op : Ops) {
471 ID.AddPointer(Op.getNode());
472 ID.AddInteger(Op.getResNo());
473 }
474}
475
476static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
477 SDVTList VTList, ArrayRef<SDValue> OpList) {
478 AddNodeIDOpcode(ID, OpC);
479 AddNodeIDValueTypes(ID, VTList);
480 AddNodeIDOperands(ID, OpList);
481}
482
483/// If this is an SDNode with special info, add this info to the NodeID data.
484static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
485 switch (N->getOpcode()) {
486 case ISD::TargetExternalSymbol:
487 case ISD::ExternalSymbol:
488 case ISD::MCSymbol:
489 llvm_unreachable("Should only be used on nodes with operands");
490 default: break; // Normal nodes don't need extra info.
491 case ISD::TargetConstant:
492 case ISD::Constant: {
493 const ConstantSDNode *C = cast<ConstantSDNode>(N);
494 ID.AddPointer(C->getConstantIntValue());
495 ID.AddBoolean(C->isOpaque());
496 break;
497 }
498 case ISD::TargetConstantFP:
499 case ISD::ConstantFP:
500 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
501 break;
502 case ISD::TargetGlobalAddress:
503 case ISD::GlobalAddress:
504 case ISD::TargetGlobalTLSAddress:
505 case ISD::GlobalTLSAddress: {
506 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
507 ID.AddPointer(GA->getGlobal());
508 ID.AddInteger(GA->getOffset());
509 ID.AddInteger(GA->getTargetFlags());
510 break;
511 }
512 case ISD::BasicBlock:
513 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
514 break;
515 case ISD::Register:
516 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
517 break;
518 case ISD::RegisterMask:
519 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
520 break;
521 case ISD::SRCVALUE:
522 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
523 break;
524 case ISD::FrameIndex:
525 case ISD::TargetFrameIndex:
526 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
527 break;
528 case ISD::LIFETIME_START:
529 case ISD::LIFETIME_END:
530 if (cast<LifetimeSDNode>(N)->hasOffset()) {
531 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
532 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
533 }
534 break;
535 case ISD::JumpTable:
536 case ISD::TargetJumpTable:
537 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
538 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
539 break;
540 case ISD::ConstantPool:
541 case ISD::TargetConstantPool: {
542 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
543 ID.AddInteger(CP->getAlignment());
544 ID.AddInteger(CP->getOffset());
545 if (CP->isMachineConstantPoolEntry())
546 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
547 else
548 ID.AddPointer(CP->getConstVal());
549 ID.AddInteger(CP->getTargetFlags());
550 break;
551 }
552 case ISD::TargetIndex: {
553 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
554 ID.AddInteger(TI->getIndex());
555 ID.AddInteger(TI->getOffset());
556 ID.AddInteger(TI->getTargetFlags());
557 break;
558 }
559 case ISD::LOAD: {
560 const LoadSDNode *LD = cast<LoadSDNode>(N);
561 ID.AddInteger(LD->getMemoryVT().getRawBits());
562 ID.AddInteger(LD->getRawSubclassData());
563 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
564 break;
565 }
566 case ISD::STORE: {
567 const StoreSDNode *ST = cast<StoreSDNode>(N);
568 ID.AddInteger(ST->getMemoryVT().getRawBits());
569 ID.AddInteger(ST->getRawSubclassData());
570 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
571 break;
572 }
573 case ISD::MLOAD: {
574 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
575 ID.AddInteger(MLD->getMemoryVT().getRawBits());
576 ID.AddInteger(MLD->getRawSubclassData());
577 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
578 break;
579 }
580 case ISD::MSTORE: {
581 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
582 ID.AddInteger(MST->getMemoryVT().getRawBits());
583 ID.AddInteger(MST->getRawSubclassData());
584 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
585 break;
586 }
587 case ISD::MGATHER: {
588 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
589 ID.AddInteger(MG->getMemoryVT().getRawBits());
590 ID.AddInteger(MG->getRawSubclassData());
591 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
592 break;
593 }
594 case ISD::MSCATTER: {
595 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
596 ID.AddInteger(MS->getMemoryVT().getRawBits());
597 ID.AddInteger(MS->getRawSubclassData());
598 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
599 break;
600 }
601 case ISD::ATOMIC_CMP_SWAP:
602 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
603 case ISD::ATOMIC_SWAP:
604 case ISD::ATOMIC_LOAD_ADD:
605 case ISD::ATOMIC_LOAD_SUB:
606 case ISD::ATOMIC_LOAD_AND:
607 case ISD::ATOMIC_LOAD_CLR:
608 case ISD::ATOMIC_LOAD_OR:
609 case ISD::ATOMIC_LOAD_XOR:
610 case ISD::ATOMIC_LOAD_NAND:
611 case ISD::ATOMIC_LOAD_MIN:
612 case ISD::ATOMIC_LOAD_MAX:
613 case ISD::ATOMIC_LOAD_UMIN:
614 case ISD::ATOMIC_LOAD_UMAX:
615 case ISD::ATOMIC_LOAD:
616 case ISD::ATOMIC_STORE: {
617 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
618 ID.AddInteger(AT->getMemoryVT().getRawBits());
619 ID.AddInteger(AT->getRawSubclassData());
620 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
621 break;
622 }
623 case ISD::PREFETCH: {
624 const MemSDNode *PF = cast<MemSDNode>(N);
625 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
626 break;
627 }
628 case ISD::VECTOR_SHUFFLE: {
629 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
630 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
631 i != e; ++i)
632 ID.AddInteger(SVN->getMaskElt(i));
633 break;
634 }
635 case ISD::TargetBlockAddress:
636 case ISD::BlockAddress: {
637 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
638 ID.AddPointer(BA->getBlockAddress());
639 ID.AddInteger(BA->getOffset());
640 ID.AddInteger(BA->getTargetFlags());
641 break;
642 }
643 } // end switch (N->getOpcode())
644
645 // Target specific memory nodes could also have address spaces to check.
646 if (N->isTargetMemoryOpcode())
647 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
648}
649
650/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
651/// data.
652static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
653 AddNodeIDOpcode(ID, N->getOpcode());
654 // Add the return value info.
655 AddNodeIDValueTypes(ID, N->getVTList());
656 // Add the operand info.
657 AddNodeIDOperands(ID, N->ops());
658
659 // Handle SDNode leafs with special info.
660 AddNodeIDCustom(ID, N);
661}
662
663//===----------------------------------------------------------------------===//
664// SelectionDAG Class
665//===----------------------------------------------------------------------===//
666
667/// doNotCSE - Return true if CSE should not be performed for this node.
668static bool doNotCSE(SDNode *N) {
669 if (N->getValueType(0) == MVT::Glue)
670 return true; // Never CSE anything that produces a flag.
671
672 switch (N->getOpcode()) {
673 default: break;
674 case ISD::HANDLENODE:
675 case ISD::EH_LABEL:
676 return true; // Never CSE these nodes.
677 }
678
679 // Check that remaining values produced are not flags.
680 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
681 if (N->getValueType(i) == MVT::Glue)
682 return true; // Never CSE anything that produces a flag.
683
684 return false;
685}
686
687/// RemoveDeadNodes - This method deletes all unreachable nodes in the
688/// SelectionDAG.
689void SelectionDAG::RemoveDeadNodes() {
690 // Create a dummy node (which is not added to allnodes), that adds a reference
691 // to the root node, preventing it from being deleted.
692 HandleSDNode Dummy(getRoot());
693
694 SmallVector<SDNode*, 128> DeadNodes;
695
696 // Add all obviously-dead nodes to the DeadNodes worklist.
697 for (SDNode &Node : allnodes())
698 if (Node.use_empty())
699 DeadNodes.push_back(&Node);
700
701 RemoveDeadNodes(DeadNodes);
702
703 // If the root changed (e.g. it was a dead load, update the root).
704 setRoot(Dummy.getValue());
705}
706
707/// RemoveDeadNodes - This method deletes the unreachable nodes in the
708/// given list, and any nodes that become unreachable as a result.
709void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
710
711 // Process the worklist, deleting the nodes and adding their uses to the
712 // worklist.
713 while (!DeadNodes.empty()) {
714 SDNode *N = DeadNodes.pop_back_val();
715 // Skip to next node if we've already managed to delete the node. This could
716 // happen if replacing a node causes a node previously added to the node to
717 // be deleted.
718 if (N->getOpcode() == ISD::DELETED_NODE)
719 continue;
720
721 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
722 DUL->NodeDeleted(N, nullptr);
723
724 // Take the node out of the appropriate CSE map.
725 RemoveNodeFromCSEMaps(N);
726
727 // Next, brutally remove the operand list. This is safe to do, as there are
728 // no cycles in the graph.
729 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
730 SDUse &Use = *I++;
731 SDNode *Operand = Use.getNode();
732 Use.set(SDValue());
733
734 // Now that we removed this operand, see if there are no uses of it left.
735 if (Operand->use_empty())
736 DeadNodes.push_back(Operand);
737 }
738
739 DeallocateNode(N);
740 }
741}
742
743void SelectionDAG::RemoveDeadNode(SDNode *N){
744 SmallVector<SDNode*, 16> DeadNodes(1, N);
745
746 // Create a dummy node that adds a reference to the root node, preventing
747 // it from being deleted. (This matters if the root is an operand of the
748 // dead node.)
749 HandleSDNode Dummy(getRoot());
750
751 RemoveDeadNodes(DeadNodes);
752}
753
754void SelectionDAG::DeleteNode(SDNode *N) {
755 // First take this out of the appropriate CSE map.
756 RemoveNodeFromCSEMaps(N);
757
758 // Finally, remove uses due to operands of this node, remove from the
759 // AllNodes list, and delete the node.
760 DeleteNodeNotInCSEMaps(N);
761}
762
763void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
764 assert(N->getIterator() != AllNodes.begin() &&
765 "Cannot delete the entry node!");
766 assert(N->use_empty() && "Cannot delete a node that is not dead!");
767
768 // Drop all of the operands and decrement used node's use counts.
769 N->DropOperands();
770
771 DeallocateNode(N);
772}
773
774void SDDbgInfo::erase(const SDNode *Node) {
775 DbgValMapType::iterator I = DbgValMap.find(Node);
776 if (I == DbgValMap.end())
777 return;
778 for (auto &Val: I->second)
779 Val->setIsInvalidated();
780 DbgValMap.erase(I);
781}
782
783void SelectionDAG::DeallocateNode(SDNode *N) {
784 // If we have operands, deallocate them.
785 removeOperands(N);
786
787 NodeAllocator.Deallocate(AllNodes.remove(N));
788
789 // Set the opcode to DELETED_NODE to help catch bugs when node
790 // memory is reallocated.
791 // FIXME: There are places in SDag that have grown a dependency on the opcode
792 // value in the released node.
793 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
794 N->NodeType = ISD::DELETED_NODE;
795
796 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
797 // them and forget about that node.
798 DbgInfo->erase(N);
799}
800
801#ifndef NDEBUG
802/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
803static void VerifySDNode(SDNode *N) {
804 switch (N->getOpcode()) {
805 default:
806 break;
807 case ISD::BUILD_PAIR: {
808 EVT VT = N->getValueType(0);
809 assert(N->getNumValues() == 1 && "Too many results!");
810 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
811 "Wrong return type!");
812 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
813 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
814 "Mismatched operand types!");
815 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
816 "Wrong operand type!");
817 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
818 "Wrong return type size");
819 break;
820 }
821 case ISD::BUILD_VECTOR: {
822 assert(N->getNumValues() == 1 && "Too many results!");
823 assert(N->getValueType(0).isVector() && "Wrong return type!");
824 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
825 "Wrong number of operands!");
826 EVT EltVT = N->getValueType(0).getVectorElementType();
827 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
828 assert((I->getValueType() == EltVT ||
829 (EltVT.isInteger() && I->getValueType().isInteger() &&
830 EltVT.bitsLE(I->getValueType()))) &&
831 "Wrong operand type!");
832 assert(I->getValueType() == N->getOperand(0).getValueType() &&
833 "Operands must all have the same type");
834 }
835 break;
836 }
837 }
838}
839#endif // NDEBUG
840
841/// Insert a newly allocated node into the DAG.
842///
843/// Handles insertion into the all nodes list and CSE map, as well as
844/// verification and other common operations when a new node is allocated.
845void SelectionDAG::InsertNode(SDNode *N) {
846 AllNodes.push_back(N);
847#ifndef NDEBUG
848 N->PersistentId = NextPersistentId++;
849 VerifySDNode(N);
850#endif
851 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
852 DUL->NodeInserted(N);
853}
854
855/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
856/// correspond to it. This is useful when we're about to delete or repurpose
857/// the node. We don't want future request for structurally identical nodes
858/// to return N anymore.
859bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
860 bool Erased = false;
861 switch (N->getOpcode()) {
862 case ISD::HANDLENODE: return false; // noop.
863 case ISD::CONDCODE:
864 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
865 "Cond code doesn't exist!");
866 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
867 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
868 break;
869 case ISD::ExternalSymbol:
870 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
871 break;
872 case ISD::TargetExternalSymbol: {
873 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
874 Erased = TargetExternalSymbols.erase(
875 std::pair<std::string,unsigned char>(ESN->getSymbol(),
876 ESN->getTargetFlags()));
877 break;
878 }
879 case ISD::MCSymbol: {
880 auto *MCSN = cast<MCSymbolSDNode>(N);
881 Erased = MCSymbols.erase(MCSN->getMCSymbol());
882 break;
883 }
884 case ISD::VALUETYPE: {
885 EVT VT = cast<VTSDNode>(N)->getVT();
886 if (VT.isExtended()) {
887 Erased = ExtendedValueTypeNodes.erase(VT);
888 } else {
889 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
890 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
891 }
892 break;
893 }
894 default:
895 // Remove it from the CSE Map.
896 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
897 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
898 Erased = CSEMap.RemoveNode(N);
899 break;
900 }
901#ifndef NDEBUG
902 // Verify that the node was actually in one of the CSE maps, unless it has a
903 // flag result (which cannot be CSE'd) or is one of the special cases that are
904 // not subject to CSE.
905 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
906 !N->isMachineOpcode() && !doNotCSE(N)) {
907 N->dump(this);
908 dbgs() << "\n";
909 llvm_unreachable("Node is not in map!");
910 }
911#endif
912 return Erased;
913}
914
915/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
916/// maps and modified in place. Add it back to the CSE maps, unless an identical
917/// node already exists, in which case transfer all its users to the existing
918/// node. This transfer can potentially trigger recursive merging.
919void
920SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
921 // For node types that aren't CSE'd, just act as if no identical node
922 // already exists.
923 if (!doNotCSE(N)) {
924 SDNode *Existing = CSEMap.GetOrInsertNode(N);
925 if (Existing != N) {
926 // If there was already an existing matching node, use ReplaceAllUsesWith
927 // to replace the dead one with the existing one. This can cause
928 // recursive merging of other unrelated nodes down the line.
929 ReplaceAllUsesWith(N, Existing);
930
931 // N is now dead. Inform the listeners and delete it.
932 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
933 DUL->NodeDeleted(N, Existing);
934 DeleteNodeNotInCSEMaps(N);
935 return;
936 }
937 }
938
939 // If the node doesn't already exist, we updated it. Inform listeners.
940 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
941 DUL->NodeUpdated(N);
942}
943
944/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
945/// were replaced with those specified. If this node is never memoized,
946/// return null, otherwise return a pointer to the slot it would take. If a
947/// node already exists with these operands, the slot will be non-null.
948SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
949 void *&InsertPos) {
950 if (doNotCSE(N))
951 return nullptr;
952
953 SDValue Ops[] = { Op };
954 FoldingSetNodeID ID;
955 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
956 AddNodeIDCustom(ID, N);
957 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
958 if (Node)
959 Node->intersectFlagsWith(N->getFlags());
960 return Node;
961}
962
963/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
964/// were replaced with those specified. If this node is never memoized,
965/// return null, otherwise return a pointer to the slot it would take. If a
966/// node already exists with these operands, the slot will be non-null.
967SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
968 SDValue Op1, SDValue Op2,
969 void *&InsertPos) {
970 if (doNotCSE(N))
971 return nullptr;
972
973 SDValue Ops[] = { Op1, Op2 };
974 FoldingSetNodeID ID;
975 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
976 AddNodeIDCustom(ID, N);
977 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
978 if (Node)
979 Node->intersectFlagsWith(N->getFlags());
980 return Node;
981}
982
983/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
984/// were replaced with those specified. If this node is never memoized,
985/// return null, otherwise return a pointer to the slot it would take. If a
986/// node already exists with these operands, the slot will be non-null.
987SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
988 void *&InsertPos) {
989 if (doNotCSE(N))
990 return nullptr;
991
992 FoldingSetNodeID ID;
993 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
994 AddNodeIDCustom(ID, N);
995 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
996 if (Node)
997 Node->intersectFlagsWith(N->getFlags());
998 return Node;
999}
1000
1001unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
1002 Type *Ty = VT == MVT::iPTR ?
1003 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
1004 VT.getTypeForEVT(*getContext());
1005
1006 return getDataLayout().getABITypeAlignment(Ty);
1007}
1008
1009// EntryNode could meaningfully have debug info if we can find it...
1010SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
1011 : TM(tm), OptLevel(OL),
1012 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1013 Root(getEntryNode()) {
1014 InsertNode(&EntryNode);
1015 DbgInfo = new SDDbgInfo();
1016}
1017
1018void SelectionDAG::init(MachineFunction &NewMF,
1019 OptimizationRemarkEmitter &NewORE,
1020 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1021 LegacyDivergenceAnalysis * Divergence) {
1022 MF = &NewMF;
1023 SDAGISelPass = PassPtr;
1024 ORE = &NewORE;
1025 TLI = getSubtarget().getTargetLowering();
1026 TSI = getSubtarget().getSelectionDAGInfo();
1027 LibInfo = LibraryInfo;
1028 Context = &MF->getFunction().getContext();
1029 DA = Divergence;
1030}
1031
1032SelectionDAG::~SelectionDAG() {
1033 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1034 allnodes_clear();
1035 OperandRecycler.clear(OperandAllocator);
1036 delete DbgInfo;
1037}
1038
1039void SelectionDAG::allnodes_clear() {
1040 assert(&*AllNodes.begin() == &EntryNode);
1041 AllNodes.remove(AllNodes.begin());
1042 while (!AllNodes.empty())
1043 DeallocateNode(&AllNodes.front());
1044#ifndef NDEBUG
1045 NextPersistentId = 0;
1046#endif
1047}
1048
1049SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1050 void *&InsertPos) {
1051 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1052 if (N) {
1053 switch (N->getOpcode()) {
1054 default: break;
1055 case ISD::Constant:
1056 case ISD::ConstantFP:
1057 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1058 "debug location. Use another overload.");
1059 }
1060 }
1061 return N;
1062}
1063
1064SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1065 const SDLoc &DL, void *&InsertPos) {
1066 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1067 if (N) {
1068 switch (N->getOpcode()) {
1069 case ISD::Constant:
1070 case ISD::ConstantFP:
1071 // Erase debug location from the node if the node is used at several
1072 // different places. Do not propagate one location to all uses as it
1073 // will cause a worse single stepping debugging experience.
1074 if (N->getDebugLoc() != DL.getDebugLoc())
1075 N->setDebugLoc(DebugLoc());
1076 break;
1077 default:
1078 // When the node's point of use is located earlier in the instruction
1079 // sequence than its prior point of use, update its debug info to the
1080 // earlier location.
1081 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1082 N->setDebugLoc(DL.getDebugLoc());
1083 break;
1084 }
1085 }
1086 return N;
1087}
1088
1089void SelectionDAG::clear() {
1090 allnodes_clear();
1091 OperandRecycler.clear(OperandAllocator);
1092 OperandAllocator.Reset();
1093 CSEMap.clear();
1094
1095 ExtendedValueTypeNodes.clear();
1096 ExternalSymbols.clear();
1097 TargetExternalSymbols.clear();
1098 MCSymbols.clear();
1099 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1100 static_cast<CondCodeSDNode*>(nullptr));
1101 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1102 static_cast<SDNode*>(nullptr));
1103
1104 EntryNode.UseList = nullptr;
1105 InsertNode(&EntryNode);
1106 Root = getEntryNode();
1107 DbgInfo->clear();
1108}
1109
1110SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1111 return VT.bitsGT(Op.getValueType())
1112 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1113 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
1114}
1115
1116SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1117 return VT.bitsGT(Op.getValueType()) ?
1118 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1119 getNode(ISD::TRUNCATE, DL, VT, Op);
1120}
1121
1122SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1123 return VT.bitsGT(Op.getValueType()) ?
1124 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1125 getNode(ISD::TRUNCATE, DL, VT, Op);
1126}
1127
1128SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1129 return VT.bitsGT(Op.getValueType()) ?
1130 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1131 getNode(ISD::TRUNCATE, DL, VT, Op);
1132}
1133
1134SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1135 EVT OpVT) {
1136 if (VT.bitsLE(Op.getValueType()))
1137 return getNode(ISD::TRUNCATE, SL, VT, Op);
1138
1139 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1140 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1141}
1142
1143SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1144 assert(!VT.isVector() &&
1145 "getZeroExtendInReg should use the vector element type instead of "
1146 "the vector type!");
1147 if (Op.getValueType().getScalarType() == VT) return Op;
1148 unsigned BitWidth = Op.getScalarValueSizeInBits();
1149 APInt Imm = APInt::getLowBitsSet(BitWidth,
1150 VT.getSizeInBits());
1151 return getNode(ISD::AND, DL, Op.getValueType(), Op,
1152 getConstant(Imm, DL, Op.getValueType()));
1153}
1154
1155SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1156 // Only unsigned pointer semantics are supported right now. In the future this
1157 // might delegate to TLI to check pointer signedness.
1158 return getZExtOrTrunc(Op, DL, VT);
1159}
1160
1161SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1162 // Only unsigned pointer semantics are supported right now. In the future this
1163 // might delegate to TLI to check pointer signedness.
1164 return getZeroExtendInReg(Op, DL, VT);
1165}
1166
1167/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1168SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1169 EVT EltVT = VT.getScalarType();
1170 SDValue NegOne =
1171 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1172 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1173}
1174
1175SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1176 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1177 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1178}
1179
1180SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1181 EVT OpVT) {
1182 if (!V)
1183 return getConstant(0, DL, VT);
1184
1185 switch (TLI->getBooleanContents(OpVT)) {
1186 case TargetLowering::ZeroOrOneBooleanContent:
1187 case TargetLowering::UndefinedBooleanContent:
1188 return getConstant(1, DL, VT);
1189 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1190 return getAllOnesConstant(DL, VT);
1191 }
1192 llvm_unreachable("Unexpected boolean content enum!");
1193}
1194
1195SDValue SelectionDAG::getNullCapability(const SDLoc &DL, EVT CapType) {
1196 assert(CapType.isFatPointer());
1197 MVT IntVT = MVT::getIntegerVT(getDataLayout().getPointerSizeInBits(0));
1198 return getNode(ISD::INTTOPTR, DL, CapType, getConstant(0, DL, IntVT));
1199}
1200
1201SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1202 bool isT, bool isO) {
1203 EVT EltVT = VT.getScalarType();
1204 assert((EltVT.getSizeInBits() >= 64 ||
1205 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1206 "getConstant with a uint64_t value that doesn't fit in the type!");
1207 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1208}
1209
1210SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1211 bool isT, bool isO) {
1212 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1213}
1214
1215SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1216 EVT VT, bool isT, bool isO) {
1217 if (VT.isFatPointer()) {
1218 unsigned BitWidth = getDataLayout().getPointerSizeInBits(0);
1219 MVT IntVT = MVT::getIntegerVT(BitWidth);
1220 APInt Int = Val.getValue();
1221 if (Int.getBitWidth() > BitWidth)
1222 Int = Int.trunc(BitWidth);
1223 const ConstantInt *V = ConstantInt::get(*Context, Int);
1224 SDValue IntVal = getConstant(*V, DL, IntVT, isT);
1225 assert(!isT && "Cannot create INTTOPTR targetconstant");
1226 // TODO: For MIPS we could copy from CNULL for value 0
1227 return getNode(ISD::INTTOPTR, SDLoc(), VT, IntVal);
1228 }
1229 assert(VT.isInteger() && "Cannot create FP integer constant!");
1230
1231 EVT EltVT = VT.getScalarType();
1232 const ConstantInt *Elt = &Val;
1233
1234 // In some cases the vector type is legal but the element type is illegal and
1235 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1236 // inserted value (the type does not need to match the vector element type).
1237 // Any extra bits introduced will be truncated away.
1238 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1239 TargetLowering::TypePromoteInteger) {
1240 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1241 APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1242 Elt = ConstantInt::get(*getContext(), NewVal);
1243 }
1244 // In other cases the element type is illegal and needs to be expanded, for
1245 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1246 // the value into n parts and use a vector type with n-times the elements.
1247 // Then bitcast to the type requested.
1248 // Legalizing constants too early makes the DAGCombiner's job harder so we
1249 // only legalize if the DAG tells us we must produce legal types.
1250 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1251 TLI->getTypeAction(*getContext(), EltVT) ==
1252 TargetLowering::TypeExpandInteger) {
1253 const APInt &NewVal = Elt->getValue();
1254 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1255 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1256 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1257 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1258
1259 // Check the temporary vector is the correct size. If this fails then
1260 // getTypeToTransformTo() probably returned a type whose size (in bits)
1261 // isn't a power-of-2 factor of the requested type size.
1262 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1263
1264 SmallVector<SDValue, 2> EltParts;
1265 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1266 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1267 .zextOrTrunc(ViaEltSizeInBits), DL,
1268 ViaEltVT, isT, isO));
1269 }
1270
1271 // EltParts is currently in little endian order. If we actually want
1272 // big-endian order then reverse it now.
1273 if (getDataLayout().isBigEndian())
1274 std::reverse(EltParts.begin(), EltParts.end());
1275
1276 // The elements must be reversed when the element order is different
1277 // to the endianness of the elements (because the BITCAST is itself a
1278 // vector shuffle in this situation). However, we do not need any code to
1279 // perform this reversal because getConstant() is producing a vector
1280 // splat.
1281 // This situation occurs in MIPS MSA.
1282
1283 SmallVector<SDValue, 8> Ops;
1284 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1285 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1286
1287 SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1288 return V;
1289 }
1290
1291 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1292 "APInt size does not match type size!");
1293 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1294 FoldingSetNodeID ID;
1295 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1296 ID.AddPointer(Elt);
1297 ID.AddBoolean(isO);
1298 void *IP = nullptr;
1299 SDNode *N = nullptr;
1300 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1301 if (!VT.isVector())
1302 return SDValue(N, 0);
1303
1304 if (!N) {
1305 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1306 CSEMap.InsertNode(N, IP);
1307 InsertNode(N);
1308 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1309 }
1310
1311 SDValue Result(N, 0);
1312 if (VT.isVector())
1313 Result = getSplatBuildVector(VT, DL, Result);
1314
1315 return Result;
1316}
1317
1318SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1319 bool isTarget) {
1320 return getConstant(Val, DL, TLI->getPointerRangeTy(getDataLayout()),
1321 isTarget);
1322}
1323
1324SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1325 const SDLoc &DL, bool LegalTypes) {
1326 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1327 return getConstant(Val, DL, ShiftVT);
1328}
1329
1330SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1331 bool isTarget) {
1332 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1333}
1334
1335SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1336 EVT VT, bool isTarget) {
1337 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1338
1339 EVT EltVT = VT.getScalarType();
1340
1341 // Do the map lookup using the actual bit pattern for the floating point
1342 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1343 // we don't have issues with SNANs.
1344 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1345 FoldingSetNodeID ID;
1346 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1347 ID.AddPointer(&V);
1348 void *IP = nullptr;
1349 SDNode *N = nullptr;
1350 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1351 if (!VT.isVector())
1352 return SDValue(N, 0);
1353
1354 if (!N) {
1355 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1356 CSEMap.InsertNode(N, IP);
1357 InsertNode(N);
1358 }
1359
1360 SDValue Result(N, 0);
1361 if (VT.isVector())
1362 Result = getSplatBuildVector(VT, DL, Result);
1363 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1364 return Result;
1365}
1366
1367SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1368 bool isTarget) {
1369 EVT EltVT = VT.getScalarType();
1370 if (EltVT == MVT::f32)
1371 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1372 else if (EltVT == MVT::f64)
1373 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1374 else if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1375 EltVT == MVT::f16) {
1376 bool Ignored;
1377 APFloat APF = APFloat(Val);
1378 APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1379 &Ignored);
1380 return getConstantFP(APF, DL, VT, isTarget);
1381 } else
1382 llvm_unreachable("Unsupported type in getConstantFP");
1383}
1384
1385SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1386 EVT VT, int64_t Offset, bool isTargetGA,
1387 unsigned char TargetFlags) {
1388 assert((TargetFlags == 0 || isTargetGA) &&
1389 "Cannot set target flags on target-independent globals");
1390
1391 // Truncate (with sign-extension) the offset value to the pointer size.
1392 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1393 if (BitWidth < 64)
1394 Offset = SignExtend64(Offset, BitWidth);
1395
1396 unsigned Opc;
1397 if (GV->isThreadLocal())
1398 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1399 else
1400 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1401
1402 FoldingSetNodeID ID;
1403 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1404 ID.AddPointer(GV);
1405 ID.AddInteger(Offset);
1406 ID.AddInteger(TargetFlags);
1407 void *IP = nullptr;
1408 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1409 return SDValue(E, 0);
1410
1411 auto *N = newSDNode<GlobalAddressSDNode>(
1412 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1413 CSEMap.InsertNode(N, IP);
1414 InsertNode(N);
1415 return SDValue(N, 0);
1416}
1417
1418SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1419 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1420 FoldingSetNodeID ID;
1421 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1422 ID.AddInteger(FI);
1423 void *IP = nullptr;
1424 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1425 return SDValue(E, 0);
1426
1427 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1428 CSEMap.InsertNode(N, IP);
1429 InsertNode(N);
1430 return SDValue(N, 0);
1431}
1432
1433SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1434 unsigned char TargetFlags) {
1435 assert((TargetFlags == 0 || isTarget) &&
1436 "Cannot set target flags on target-independent jump tables");
1437 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1438 FoldingSetNodeID ID;
1439 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1440 ID.AddInteger(JTI);
1441 ID.AddInteger(TargetFlags);
1442 void *IP = nullptr;
1443 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1444 return SDValue(E, 0);
1445
1446 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1447 CSEMap.InsertNode(N, IP);
1448 InsertNode(N);
1449 return SDValue(N, 0);
1450}
1451
1452SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1453 unsigned Alignment, int Offset,
1454 bool isTarget,
1455 unsigned char TargetFlags) {
1456 assert((TargetFlags == 0 || isTarget) &&
1457 "Cannot set target flags on target-independent globals");
1458 if (Alignment == 0)
1459 Alignment = MF->getFunction().hasOptSize()
1460 ? getDataLayout().getABITypeAlignment(C->getType())
1461 : getDataLayout().getPrefTypeAlignment(C->getType());
1462 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1463 FoldingSetNodeID ID;
1464 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1465 ID.AddInteger(Alignment);
1466 ID.AddInteger(Offset);
1467 ID.AddPointer(C);
1468 ID.AddInteger(TargetFlags);
1469 void *IP = nullptr;
1470 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1471 return SDValue(E, 0);
1472
1473 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1474 TargetFlags);
1475 CSEMap.InsertNode(N, IP);
1476 InsertNode(N);
1477 return SDValue(N, 0);
1478}
1479
1480SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1481 unsigned Alignment, int Offset,
1482 bool isTarget,
1483 unsigned char TargetFlags) {
1484 assert((TargetFlags == 0 || isTarget) &&
1485 "Cannot set target flags on target-independent globals");
1486 if (Alignment == 0)
1487 Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1488 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1489 FoldingSetNodeID ID;
1490 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1491 ID.AddInteger(Alignment);
1492 ID.AddInteger(Offset);
1493 C->addSelectionDAGCSEId(ID);
1494 ID.AddInteger(TargetFlags);
1495 void *IP = nullptr;
1496 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1497 return SDValue(E, 0);
1498
1499 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1500 TargetFlags);
1501 CSEMap.InsertNode(N, IP);
1502 InsertNode(N);
1503 return SDValue(N, 0);
1504}
1505
1506SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1507 unsigned char TargetFlags) {
1508 FoldingSetNodeID ID;
1509 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1510 ID.AddInteger(Index);
1511 ID.AddInteger(Offset);
1512 ID.AddInteger(TargetFlags);
1513 void *IP = nullptr;
1514 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1515 return SDValue(E, 0);
1516
1517 auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1518 CSEMap.InsertNode(N, IP);
1519 InsertNode(N);
1520 return SDValue(N, 0);
1521}
1522
1523SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1524 FoldingSetNodeID ID;
1525 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1526 ID.AddPointer(MBB);
1527 void *IP = nullptr;
1528 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1529 return SDValue(E, 0);
1530
1531 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1532 CSEMap.InsertNode(N, IP);
1533 InsertNode(N);
1534 return SDValue(N, 0);
1535}
1536
1537SDValue SelectionDAG::getValueType(EVT VT) {
1538 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1539 ValueTypeNodes.size())
1540 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1541
1542 SDNode *&N = VT.isExtended() ?
1543 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1544
1545 if (N) return SDValue(N, 0);
1546 N = newSDNode<VTSDNode>(VT);
1547 InsertNode(N);
1548 return SDValue(N, 0);
1549}
1550
1551SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1552 SDNode *&N = ExternalSymbols[Sym];
1553 if (N) return SDValue(N, 0);
1554 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1555 InsertNode(N);
1556 return SDValue(N, 0);
1557}
1558
1559SDValue SelectionDAG::getExternalFunctionSymbol(const char *Sym) {
1560 auto AddrSpace = getDataLayout().getProgramAddressSpace();
1561 return getExternalSymbol(Sym, TLI->getPointerTy(getDataLayout(), AddrSpace));
1562}
1563
1564SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1565 SDNode *&N = MCSymbols[Sym];
1566 if (N)
1567 return SDValue(N, 0);
1568 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1569 InsertNode(N);
1570 return SDValue(N, 0);
1571}
1572
1573SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1574 unsigned char TargetFlags) {
1575 SDNode *&N =
1576 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1577 TargetFlags)];
1578 if (N) return SDValue(N, 0);
1579 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1580 InsertNode(N);
1581 return SDValue(N, 0);
1582}
1583
1584SDValue
1585SelectionDAG::getTargetExternalFunctionSymbol(const char *Sym,
1586 unsigned char TargetFlags) {
1587 auto AddrSpace = getDataLayout().getProgramAddressSpace();
1588 return getTargetExternalSymbol(
1589 Sym, TLI->getPointerTy(getDataLayout(), AddrSpace), TargetFlags);
1590}
1591
1592SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1593 if ((unsigned)Cond >= CondCodeNodes.size())
1594 CondCodeNodes.resize(Cond+1);
1595
1596 if (!CondCodeNodes[Cond]) {
1597 auto *N = newSDNode<CondCodeSDNode>(Cond);
1598 CondCodeNodes[Cond] = N;
1599 InsertNode(N);
1600 }
1601
1602 return SDValue(CondCodeNodes[Cond], 0);
1603}
1604
1605/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1606/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1607static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1608 std::swap(N1, N2);
1609 ShuffleVectorSDNode::commuteMask(M);
1610}
1611
1612SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1613 SDValue N2, ArrayRef<int> Mask) {
1614 assert(VT.getVectorNumElements() == Mask.size() &&
1615 "Must have the same number of vector elements as mask elements!");
1616 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1617 "Invalid VECTOR_SHUFFLE");
1618
1619 // Canonicalize shuffle undef, undef -> undef
1620 if (N1.isUndef() && N2.isUndef())
1621 return getUNDEF(VT);
1622
1623 // Validate that all indices in Mask are within the range of the elements
1624 // input to the shuffle.
1625 int NElts = Mask.size();
1626 assert(llvm::all_of(Mask,
1627 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
1628 "Index out of range");
1629
1630 // Copy the mask so we can do any needed cleanup.
1631 SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1632
1633 // Canonicalize shuffle v, v -> v, undef
1634 if (N1 == N2) {
1635 N2 = getUNDEF(VT);
1636 for (int i = 0; i != NElts; ++i)
1637 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
1638 }
1639
1640 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1641 if (N1.isUndef())
1642 commuteShuffle(N1, N2, MaskVec);
1643
1644 if (TLI->hasVectorBlend()) {
1645 // If shuffling a splat, try to blend the splat instead. We do this here so
1646 // that even when this arises during lowering we don't have to re-handle it.
1647 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1648 BitVector UndefElements;
1649 SDValue Splat = BV->getSplatValue(&UndefElements);
1650 if (!Splat)
1651 return;
1652
1653 for (int i = 0; i < NElts; ++i) {
1654 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
1655 continue;
1656
1657 // If this input comes from undef, mark it as such.
1658 if (UndefElements[MaskVec[i] - Offset]) {
1659 MaskVec[i] = -1;
1660 continue;
1661 }
1662
1663 // If we can blend a non-undef lane, use that instead.
1664 if (!UndefElements[i])
1665 MaskVec[i] = i + Offset;
1666 }
1667 };
1668 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1669 BlendSplat(N1BV, 0);
1670 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1671 BlendSplat(N2BV, NElts);
1672 }
1673
1674 // Canonicalize all index into lhs, -> shuffle lhs, undef
1675 // Canonicalize all index into rhs, -> shuffle rhs, undef
1676 bool AllLHS = true, AllRHS = true;
1677 bool N2Undef = N2.isUndef();
1678 for (int i = 0; i != NElts; ++i) {
1679 if (MaskVec[i] >= NElts) {
1680 if (N2Undef)
1681 MaskVec[i] = -1;
1682 else
1683 AllLHS = false;
1684 } else if (MaskVec[i] >= 0) {
1685 AllRHS = false;
1686 }
1687 }
1688 if (AllLHS && AllRHS)
1689 return getUNDEF(VT);
1690 if (AllLHS && !N2Undef)
1691 N2 = getUNDEF(VT);
1692 if (AllRHS) {
1693 N1 = getUNDEF(VT);
1694 commuteShuffle(N1, N2, MaskVec);
1695 }
1696 // Reset our undef status after accounting for the mask.
1697 N2Undef = N2.isUndef();
1698 // Re-check whether both sides ended up undef.
1699 if (N1.isUndef() && N2Undef)
1700 return getUNDEF(VT);
1701
1702 // If Identity shuffle return that node.
1703 bool Identity = true, AllSame = true;
1704 for (int i = 0; i != NElts; ++i) {
1705 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
1706 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1707 }
1708 if (Identity && NElts)
1709 return N1;
1710
1711 // Shuffling a constant splat doesn't change the result.
1712 if (N2Undef) {
1713 SDValue V = N1;
1714
1715 // Look through any bitcasts. We check that these don't change the number
1716 // (and size) of elements and just changes their types.
1717 while (V.getOpcode() == ISD::BITCAST)
1718 V = V->getOperand(0);
1719
1720 // A splat should always show up as a build vector node.
1721 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1722 BitVector UndefElements;
1723 SDValue Splat = BV->getSplatValue(&UndefElements);
1724 // If this is a splat of an undef, shuffling it is also undef.
1725 if (Splat && Splat.isUndef())
1726 return getUNDEF(VT);
1727
1728 bool SameNumElts =
1729 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1730
1731 // We only have a splat which can skip shuffles if there is a splatted
1732 // value and no undef lanes rearranged by the shuffle.
1733 if (Splat && UndefElements.none()) {
1734 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1735 // number of elements match or the value splatted is a zero constant.
1736 if (SameNumElts)
1737 return N1;
1738 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1739 if (C->isNullValue())
1740 return N1;
1741 }
1742
1743 // If the shuffle itself creates a splat, build the vector directly.
1744 if (AllSame && SameNumElts) {
1745 EVT BuildVT = BV->getValueType(0);
1746 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1747 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1748
1749 // We may have jumped through bitcasts, so the type of the
1750 // BUILD_VECTOR may not match the type of the shuffle.
1751 if (BuildVT != VT)
1752 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1753 return NewBV;
1754 }
1755 }
1756 }
1757
1758 FoldingSetNodeID ID;
1759 SDValue Ops[2] = { N1, N2 };
1760 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1761 for (int i = 0; i != NElts; ++i)
1762 ID.AddInteger(MaskVec[i]);
1763
1764 void* IP = nullptr;
1765 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1766 return SDValue(E, 0);
1767
1768 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1769 // SDNode doesn't have access to it. This memory will be "leaked" when
1770 // the node is deallocated, but recovered when the NodeAllocator is released.
1771 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1772 llvm::copy(MaskVec, MaskAlloc);
1773
1774 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1775 dl.getDebugLoc(), MaskAlloc);
1776 createOperands(N, Ops);
1777
1778 CSEMap.InsertNode(N, IP);
1779 InsertNode(N);
1780 SDValue V = SDValue(N, 0);
1781 NewSDValueDbgMsg(V, "Creating new node: ", this);
1782 return V;
1783}
1784
1785SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1786 EVT VT = SV.getValueType(0);
1787 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1788 ShuffleVectorSDNode::commuteMask(MaskVec);
1789
1790 SDValue Op0 = SV.getOperand(0);
1791 SDValue Op1 = SV.getOperand(1);
1792 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1793}
1794
1795SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1796 FoldingSetNodeID ID;
1797 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1798 ID.AddInteger(RegNo);
1799 void *IP = nullptr;
1800 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1801 return SDValue(E, 0);
1802
1803 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1804 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1805 CSEMap.InsertNode(N, IP);
1806 InsertNode(N);
1807 return SDValue(N, 0);
1808}
1809
1810SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1811 FoldingSetNodeID ID;
1812 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1813 ID.AddPointer(RegMask);
1814 void *IP = nullptr;
1815 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1816 return SDValue(E, 0);
1817
1818 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1819 CSEMap.InsertNode(N, IP);
1820 InsertNode(N);
1821 return SDValue(N, 0);
1822}
1823
1824SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1825 MCSymbol *Label) {
1826 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1827}
1828
1829SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1830 SDValue Root, MCSymbol *Label) {
1831 FoldingSetNodeID ID;
1832 SDValue Ops[] = { Root };
1833 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1834 ID.AddPointer(Label);
1835 void *IP = nullptr;
1836 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1837 return SDValue(E, 0);
1838
1839 auto *N =
1840 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
1841 createOperands(N, Ops);
1842
1843 CSEMap.InsertNode(N, IP);
1844 InsertNode(N);
1845 return SDValue(N, 0);
1846}
1847
1848SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1849 int64_t Offset,
1850 bool isTarget,
1851 unsigned char TargetFlags) {
1852 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1853
1854 FoldingSetNodeID ID;
1855 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1856 ID.AddPointer(BA);
1857 ID.AddInteger(Offset);
1858 ID.AddInteger(TargetFlags);
1859 void *IP = nullptr;
1860 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1861 return SDValue(E, 0);
1862
1863 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1864 CSEMap.InsertNode(N, IP);
1865 InsertNode(N);
1866 return SDValue(N, 0);
1867}
1868
1869SDValue SelectionDAG::getSrcValue(const Value *V) {
1870 assert((!V || V->getType()->isPointerTy()) &&
1871 "SrcValue is not a pointer?");
1872
1873 FoldingSetNodeID ID;
1874 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1875 ID.AddPointer(V);
1876
1877 void *IP = nullptr;
1878 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1879 return SDValue(E, 0);
1880
1881 auto *N = newSDNode<SrcValueSDNode>(V);
1882 CSEMap.InsertNode(N, IP);
1883 InsertNode(N);
1884 return SDValue(N, 0);
1885}
1886
1887SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1888 FoldingSetNodeID ID;
1889 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1890 ID.AddPointer(MD);
1891
1892 void *IP = nullptr;
1893 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1894 return SDValue(E, 0);
1895
1896 auto *N = newSDNode<MDNodeSDNode>(MD);
1897 CSEMap.InsertNode(N, IP);
1898 InsertNode(N);
1899 return SDValue(N, 0);
1900}
1901
1902SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1903 if (VT == V.getValueType())
1904 return V;
1905
1906 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1907}
1908
1909SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1910 unsigned SrcAS, unsigned DestAS) {
1911 SDValue Ops[] = {Ptr};
1912 FoldingSetNodeID ID;
1913 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1914 ID.AddInteger(SrcAS);
1915 ID.AddInteger(DestAS);
1916
1917 void *IP = nullptr;
1918 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1919 return SDValue(E, 0);
1920
1921 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1922 VT, SrcAS, DestAS);
1923 createOperands(N, Ops);
1924
1925 CSEMap.InsertNode(N, IP);
1926 InsertNode(N);
1927 return SDValue(N, 0);
1928}
1929
1930/// getShiftAmountOperand - Return the specified value casted to
1931/// the target's desired shift amount type.
1932SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1933 EVT OpTy = Op.getValueType();
1934 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1935 if (OpTy == ShTy || OpTy.isVector()) return Op;
1936
1937 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1938}
1939
1940SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1941 SDLoc dl(Node);
1942 const TargetLowering &TLI = getTargetLoweringInfo();
1943 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1944 EVT VT = Node->getValueType(0);
1945 SDValue Tmp1 = Node->getOperand(0);
1946 SDValue Tmp2 = Node->getOperand(1);
1947 unsigned Align = Node->getConstantOperandVal(3);
1948
1949 SDValue VAListLoad = getLoad(
1950 TLI.getPointerTy(getDataLayout(), getDataLayout().getAllocaAddrSpace()),
1951 dl, Tmp1, Tmp2, MachinePointerInfo(V));
1952 SDValue VAList = VAListLoad;
1953
1954 if (Align > TLI.getMinStackArgumentAlignment()) {
1955 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
1956
1957 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1958 getConstant(Align - 1, dl, VAList.getValueType()));
1959
1960 VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1961 getConstant(-(int64_t)Align, dl, VAList.getValueType()));
1962 }
1963
1964 // Increment the pointer, VAList, to the next vaarg
1965 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1966 getConstant(getDataLayout().getTypeAllocSize(
1967 VT.getTypeForEVT(*getContext())),
1968 dl, VAList.getValueType()));
1969 // Store the incremented VAList to the legalized pointer
1970 Tmp1 =
1971 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
1972 // Load the actual argument out of the pointer VAList
1973 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
1974}
1975
1976SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1977 SDLoc dl(Node);
1978 const TargetLowering &TLI = getTargetLoweringInfo();
1979 // This defaults to loading a pointer from the input and storing it to the
1980 // output, returning the chain.
1981 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1982 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1983 SDValue Tmp1 = getLoad(
1984 TLI.getPointerTy(getDataLayout(), getDataLayout().getAllocaAddrSpace()),
1985 dl, Node->getOperand(0), Node->getOperand(2), MachinePointerInfo(VS));
1986 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1987 MachinePointerInfo(VD));
1988}
1989
1990SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1991 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1992 unsigned ByteSize = VT.getStoreSize();
1993 Type *Ty = VT.getTypeForEVT(*getContext());
1994 unsigned StackAlign =
1995 std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1996
1997 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
1998 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1999}
2000
2001SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2002 unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
2003 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2004 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2005 const DataLayout &DL = getDataLayout();
2006 unsigned Align =
2007 std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
2008
2009 MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
2010 int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
2011 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2012}
2013
2014SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2015 ISD::CondCode Cond, const SDLoc &dl) {
2016 EVT OpVT = N1.getValueType();
2017
2018 // These setcc operations always fold.
2019 switch (Cond) {
2020 default: break;
2021 case ISD::SETFALSE:
2022 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2023 case ISD::SETTRUE:
2024 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2025
2026 case ISD::SETOEQ:
2027 case ISD::SETOGT:
2028 case ISD::SETOGE:
2029 case ISD::SETOLT:
2030 case ISD::SETOLE:
2031 case ISD::SETONE:
2032 case ISD::SETO:
2033 case ISD::SETUO:
2034 case ISD::SETUEQ:
2035 case ISD::SETUNE:
2036 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2037 break;
2038 }
2039
2040 if (OpVT.isInteger()) {
2041 // For EQ and NE, we can always pick a value for the undef to make the
2042 // predicate pass or fail, so we can return undef.
2043 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2044 // icmp eq/ne X, undef -> undef.
2045 if ((N1.isUndef() || N2.isUndef()) &&
2046 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2047 return getUNDEF(VT);
2048
2049 // If both operands are undef, we can return undef for int comparison.
2050 // icmp undef, undef -> undef.
2051 if (N1.isUndef() && N2.isUndef())
2052 return getUNDEF(VT);
2053
2054 // icmp X, X -> true/false
2055 // icmp X, undef -> true/false because undef could be X.
2056 if (N1 == N2)
2057 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2058 }
2059
2060 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2061 const APInt &C2 = N2C->getAPIntValue();
2062 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2063 const APInt &C1 = N1C->getAPIntValue();
2064
2065 switch (Cond) {
2066 default: llvm_unreachable("Unknown integer setcc!");
2067 case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT);
2068 case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT);
2069 case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT);
2070 case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT);
2071 case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT);
2072 case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT);
2073 case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT);
2074 case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT);
2075 case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT);
2076 case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT);
2077 }
2078 }
2079 }
2080
2081 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2082 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2083
2084 if (N1CFP && N2CFP) {
2085 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2086 switch (Cond) {
2087 default: break;
2088 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2089 return getUNDEF(VT);
2090 LLVM_FALLTHROUGH;
2091 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2092 OpVT);
2093 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2094 return getUNDEF(VT);
2095 LLVM_FALLTHROUGH;
2096 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2097 R==APFloat::cmpLessThan, dl, VT,
2098 OpVT);
2099 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2100 return getUNDEF(VT);
2101 LLVM_FALLTHROUGH;
2102 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2103 OpVT);
2104 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2105 return getUNDEF(VT);
2106 LLVM_FALLTHROUGH;
2107 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2108 VT, OpVT);
2109 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2110 return getUNDEF(VT);
2111 LLVM_FALLTHROUGH;
2112 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2113 R==APFloat::cmpEqual, dl, VT,
2114 OpVT);
2115 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2116 return getUNDEF(VT);
2117 LLVM_FALLTHROUGH;
2118 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2119 R==APFloat::cmpEqual, dl, VT, OpVT);
2120 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2121 OpVT);
2122 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2123 OpVT);
2124 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2125 R==APFloat::cmpEqual, dl, VT,
2126 OpVT);
2127 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2128 OpVT);
2129 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2130 R==APFloat::cmpLessThan, dl, VT,
2131 OpVT);
2132 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2133 R==APFloat::cmpUnordered, dl, VT,
2134 OpVT);
2135 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2136 VT, OpVT);
2137 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2138 OpVT);
2139 }
2140 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2141 // Ensure that the constant occurs on the RHS.
2142 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2143 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2144 return SDValue();
2145 return getSetCC(dl, VT, N2, N1, SwappedCond);
2146 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2147 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2148 // If an operand is known to be a nan (or undef that could be a nan), we can
2149 // fold it.
2150 // Choosing NaN for the undef will always make unordered comparison succeed
2151 // and ordered comparison fails.
2152 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2153 switch (ISD::getUnorderedFlavor(Cond)) {
2154 default:
2155 llvm_unreachable("Unknown flavor!");
2156 case 0: // Known false.
2157 return getBoolConstant(false, dl, VT, OpVT);
2158 case 1: // Known true.
2159 return getBoolConstant(true, dl, VT, OpVT);
2160 case 2: // Undefined.
2161 return getUNDEF(VT);
2162 }
2163 }
2164
2165 // Could not fold it.
2166 return SDValue();
2167}
2168
2169/// See if the specified operand can be simplified with the knowledge that only
2170/// the bits specified by DemandedBits are used.
2171/// TODO: really we should be making this into the DAG equivalent of
2172/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2173SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2174 EVT VT = V.getValueType();
2175 APInt DemandedElts = VT.isVector()
2176 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2177 : APInt(1, 1);
2178 return GetDemandedBits(V, DemandedBits, DemandedElts);
2179}
2180
2181/// See if the specified operand can be simplified with the knowledge that only
2182/// the bits specified by DemandedBits are used in the elements specified by
2183/// DemandedElts.
2184/// TODO: really we should be making this into the DAG equivalent of
2185/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2186SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2187 const APInt &DemandedElts) {
2188 switch (V.getOpcode()) {
2189 default:
2190 break;
2191 case ISD::Constant: {
2192 auto *CV = cast<ConstantSDNode>(V.getNode());
2193 assert(CV && "Const value should be ConstSDNode.");
2194 const APInt &CVal = CV->getAPIntValue();
2195 APInt NewVal = CVal & DemandedBits;
2196 if (NewVal != CVal)
2197 return getConstant(NewVal, SDLoc(V), V.getValueType());
2198 break;
2199 }
2200 case ISD::OR:
2201 case ISD::XOR:
2202 // If the LHS or RHS don't contribute bits to the or, drop them.
2203 if (MaskedValueIsZero(V.getOperand(0), DemandedBits))
2204 return V.getOperand(1);
2205 if (MaskedValueIsZero(V.getOperand(1), DemandedBits))
2206 return V.getOperand(0);
2207 break;
2208 case ISD::SRL:
2209 // Only look at single-use SRLs.
2210 if (!V.getNode()->hasOneUse())
2211 break;
2212 if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2213 // See if we can recursively simplify the LHS.
2214 unsigned Amt = RHSC->getZExtValue();
2215
2216 // Watch out for shift count overflow though.
2217 if (Amt >= DemandedBits.getBitWidth())
2218 break;
2219 APInt SrcDemandedBits = DemandedBits << Amt;
2220 if (SDValue SimplifyLHS =
2221 GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2222 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2223 V.getOperand(1));
2224 }
2225 break;
2226 case ISD::AND: {
2227 // X & -1 -> X (ignoring bits which aren't demanded).
2228 // Also handle the case where masked out bits in X are known to be zero.
2229 if (ConstantSDNode *RHSC = isConstOrConstSplat(V.getOperand(1))) {
2230 const APInt &AndVal = RHSC->getAPIntValue();
2231 if (DemandedBits.isSubsetOf(AndVal) ||
2232 DemandedBits.isSubsetOf(computeKnownBits(V.getOperand(0)).Zero |
2233 AndVal))
2234 return V.getOperand(0);
2235 }
2236 break;
2237 }
2238 case ISD::ANY_EXTEND: {
2239 SDValue Src = V.getOperand(0);
2240 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
2241 // Being conservative here - only peek through if we only demand bits in the
2242 // non-extended source (even though the extended bits are technically
2243 // undef).
2244 if (DemandedBits.getActiveBits() > SrcBitWidth)
2245 break;
2246 APInt SrcDemandedBits = DemandedBits.trunc(SrcBitWidth);
2247 if (SDValue DemandedSrc = GetDemandedBits(Src, SrcDemandedBits))
2248 return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
2249 break;
2250 }
2251 case ISD::SIGN_EXTEND_INREG:
2252 EVT ExVT = cast<VTSDNode>(V.getOperand(1))->getVT();
2253 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2254
2255 // If none of the extended bits are demanded, eliminate the sextinreg.
2256 if (DemandedBits.getActiveBits() <= ExVTBits)
2257 return V.getOperand(0);
2258
2259 break;
2260 }
2261 return SDValue();
2262}
2263
2264/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2265/// use this predicate to simplify operations downstream.
2266bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2267 unsigned BitWidth = Op.getScalarValueSizeInBits();
2268 return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2269}
2270
2271/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2272/// this predicate to simplify operations downstream. Mask is known to be zero
2273/// for bits that V cannot have.
2274bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2275 unsigned Depth) const {
2276 EVT VT = V.getValueType();
2277 APInt DemandedElts = VT.isVector()
2278 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2279 : APInt(1, 1);
2280 return MaskedValueIsZero(V, Mask, DemandedElts, Depth);
2281}
2282
2283/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2284/// DemandedElts. We use this predicate to simplify operations downstream.
2285/// Mask is known to be zero for bits that V cannot have.
2286bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2287 const APInt &DemandedElts,
2288 unsigned Depth) const {
2289 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2290}
2291
2292/// isSplatValue - Return true if the vector V has the same value
2293/// across all DemandedElts.
2294bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2295 APInt &UndefElts) {
2296 if (!DemandedElts)
2297 return false; // No demanded elts, better to assume we don't know anything.
2298
2299 EVT VT = V.getValueType();
2300 assert(VT.isVector() && "Vector type expected");
2301
2302 unsigned NumElts = VT.getVectorNumElements();
2303 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2304 UndefElts = APInt::getNullValue(NumElts);
2305
2306 switch (V.getOpcode()) {
2307 case ISD::BUILD_VECTOR: {
2308 SDValue Scl;
2309 for (unsigned i = 0; i != NumElts; ++i) {
2310 SDValue Op = V.getOperand(i);
2311 if (Op.isUndef()) {
2312 UndefElts.setBit(i);
2313 continue;
2314 }
2315 if (!DemandedElts[i])
2316 continue;
2317 if (Scl && Scl != Op)
2318 return false;
2319 Scl = Op;
2320 }
2321 return true;
2322 }
2323 case ISD::VECTOR_SHUFFLE: {
2324 // Check if this is a shuffle node doing a splat.
2325 // TODO: Do we need to handle shuffle(splat, undef, mask)?
2326 int SplatIndex = -1;
2327 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2328 for (int i = 0; i != (int)NumElts; ++i) {
2329 int M = Mask[i];
2330 if (M < 0) {
2331 UndefElts.setBit(i);
2332 continue;
2333 }
2334 if (!DemandedElts[i])
2335 continue;
2336 if (0 <= SplatIndex && SplatIndex != M)
2337 return false;
2338 SplatIndex = M;
2339 }
2340 return true;
2341 }
2342 case ISD::EXTRACT_SUBVECTOR: {
2343 SDValue Src = V.getOperand(0);
2344 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(V.getOperand(1));
2345 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2346 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2347 // Offset the demanded elts by the subvector index.
2348 uint64_t Idx = SubIdx->getZExtValue();
2349 APInt UndefSrcElts;
2350 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2351 if (isSplatValue(Src, DemandedSrc, UndefSrcElts)) {
2352 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2353 return true;
2354 }
2355 }
2356 break;
2357 }
2358 case ISD::ADD:
2359 case ISD::SUB:
2360 case ISD::AND: {
2361 APInt UndefLHS, UndefRHS;
2362 SDValue LHS = V.getOperand(0);
2363 SDValue RHS = V.getOperand(1);
2364 if (isSplatValue(LHS, DemandedElts, UndefLHS) &&
2365 isSplatValue(RHS, DemandedElts, UndefRHS)) {
2366 UndefElts = UndefLHS | UndefRHS;
2367 return true;
2368 }
2369 break;
2370 }
2371 }
2372
2373 return false;
2374}
2375
2376/// Helper wrapper to main isSplatValue function.
2377bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2378 EVT VT = V.getValueType();
2379 assert(VT.isVector() && "Vector type expected");
2380 unsigned NumElts = VT.getVectorNumElements();
2381
2382 APInt UndefElts;
2383 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
2384 return isSplatValue(V, DemandedElts, UndefElts) &&
2385 (AllowUndefs || !UndefElts);
2386}
2387
2388SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2389 V = peekThroughExtractSubvectors(V);
2390
2391 EVT VT = V.getValueType();
2392 unsigned Opcode = V.getOpcode();
2393 switch (Opcode) {
2394 default: {
2395 APInt UndefElts;
2396 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2397 if (isSplatValue(V, DemandedElts, UndefElts)) {
2398 // Handle case where all demanded elements are UNDEF.
2399 if (DemandedElts.isSubsetOf(UndefElts)) {
2400 SplatIdx = 0;
2401 return getUNDEF(VT);
2402 }
2403 SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2404 return V;
2405 }
2406 break;
2407 }
2408 case ISD::VECTOR_SHUFFLE: {
2409 // Check if this is a shuffle node doing a splat.
2410 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2411 // getTargetVShiftNode currently struggles without the splat source.
2412 auto *SVN = cast<ShuffleVectorSDNode>(V);
2413 if (!SVN->isSplat())
2414 break;
2415 int Idx = SVN->getSplatIndex();
2416 int NumElts = V.getValueType().getVectorNumElements();
2417 SplatIdx = Idx % NumElts;
2418 return V.getOperand(Idx / NumElts);
2419 }
2420 }
2421
2422 return SDValue();
2423}
2424
2425SDValue SelectionDAG::getSplatValue(SDValue V) {
2426 int SplatIdx;
2427 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
2428 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V),
2429 SrcVector.getValueType().getScalarType(), SrcVector,
2430 getIntPtrConstant(SplatIdx, SDLoc(V)));
2431 return SDValue();
2432}
2433
2434/// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
2435/// is less than the element bit-width of the shift node, return it.
2436static const APInt *getValidShiftAmountConstant(SDValue V) {
2437 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
2438 // Shifting more than the bitwidth is not valid.
2439 const APInt &ShAmt = SA->getAPIntValue();
2440 if (ShAmt.ult(V.getScalarValueSizeInBits()))
2441 return &ShAmt;
2442 }
2443 return nullptr;
2444}
2445
2446/// Determine which bits of Op are known to be either zero or one and return
2447/// them in Known. For vectors, the known bits are those that are shared by
2448/// every vector element.
2449KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2450 EVT VT = Op.getValueType();
2451 APInt DemandedElts = VT.isVector()
2452 ? APInt::getAllOnesValue(VT.getVectorNumElements())
2453 : APInt(1, 1);
2454 return computeKnownBits(Op, DemandedElts, Depth);
2455}
2456
2457/// Determine which bits of Op are known to be either zero or one and return
2458/// them in Known. The DemandedElts argument allows us to only collect the known
2459/// bits that are shared by the requested vector elements.
2460KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2461 unsigned Depth) const {
2462 unsigned BitWidth = Op.getScalarValueSizeInBits();
2463
2464 KnownBits Known(BitWidth); // Don't know anything.
2465
2466 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2467 // We know all of the bits for a constant!
2468 Known.One = C->getAPIntValue();
2469 Known.Zero = ~Known.One;
2470 return Known;
2471 }
2472 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2473 // We know all of the bits for a constant fp!
2474 Known.One = C->getValueAPF().bitcastToAPInt();
2475 Known.Zero = ~Known.One;
2476 return Known;
2477 }
2478
2479 if (Depth == 6)
2480 return Known; // Limit search depth.
2481
2482 KnownBits Known2;
2483 unsigned NumElts = DemandedElts.getBitWidth();
2484 assert((!Op.getValueType().isVector() ||
2485 NumElts == Op.getValueType().getVectorNumElements()) &&
2486 "Unexpected vector size");
2487
2488 if (!DemandedElts)
2489 return Known; // No demanded elts, better to assume we don't know anything.
2490
2491 unsigned Opcode = Op.getOpcode();
2492 switch (Opcode) {
2493 case ISD::BUILD_VECTOR:
2494 // Collect the known bits that are shared by every demanded vector element.
2495 Known.Zero.setAllBits(); Known.One.setAllBits();
2496 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
2497 if (!DemandedElts[i])
2498 continue;
2499
2500 SDValue SrcOp = Op.getOperand(i);
2501 Known2 = computeKnownBits(SrcOp, Depth + 1);
2502
2503 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2504 if (SrcOp.getValueSizeInBits() != BitWidth) {
2505 assert(SrcOp.getValueSizeInBits() > BitWidth &&
2506 "Expected BUILD_VECTOR implicit truncation");
2507 Known2 = Known2.trunc(BitWidth);
2508 }
2509
2510 // Known bits are the values that are shared by every demanded element.
2511 Known.One &= Known2.One;
2512 Known.Zero &= Known2.Zero;
2513
2514 // If we don't know any bits, early out.
2515 if (Known.isUnknown())
2516 break;
2517 }
2518 break;
2519 case ISD::VECTOR_SHUFFLE: {
2520 // Collect the known bits that are shared by every vector element referenced
2521 // by the shuffle.
2522 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2523 Known.Zero.setAllBits(); Known.One.setAllBits();
2524 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2525 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
2526 for (unsigned i = 0; i != NumElts; ++i) {
2527 if (!DemandedElts[i])
2528 continue;
2529
2530 int M = SVN->getMaskElt(i);
2531 if (M < 0) {
2532 // For UNDEF elements, we don't know anything about the common state of
2533 // the shuffle result.
2534 Known.resetAll();
2535 DemandedLHS.clearAllBits();
2536 DemandedRHS.clearAllBits();
2537 break;
2538 }
2539
2540 if ((unsigned)M < NumElts)
2541 DemandedLHS.setBit((unsigned)M % NumElts);
2542 else
2543 DemandedRHS.setBit((unsigned)M % NumElts);
2544 }
2545 // Known bits are the values that are shared by every demanded element.
2546 if (!!DemandedLHS) {
2547 SDValue LHS = Op.getOperand(0);
2548 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2549 Known.One &= Known2.One;
2550 Known.Zero &= Known2.Zero;
2551 }
2552 // If we don't know any bits, early out.
2553 if (Known.isUnknown())
2554 break;
2555 if (!!DemandedRHS) {
2556 SDValue RHS = Op.getOperand(1);
2557 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2558 Known.One &= Known2.One;
2559 Known.Zero &= Known2.Zero;
2560 }
2561 break;
2562 }
2563 case ISD::CONCAT_VECTORS: {
2564 // Split DemandedElts and test each of the demanded subvectors.
2565 Known.Zero.setAllBits(); Known.One.setAllBits();
2566 EVT SubVectorVT = Op.getOperand(0).getValueType();
2567 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2568 unsigned NumSubVectors = Op.getNumOperands();
2569 for (unsigned i = 0; i != NumSubVectors; ++i) {
2570 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2571 DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2572 if (!!DemandedSub) {
2573 SDValue Sub = Op.getOperand(i);
2574 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2575 Known.One &= Known2.One;
2576 Known.Zero &= Known2.Zero;
2577 }
2578 // If we don't know any bits, early out.
2579 if (Known.isUnknown())
2580 break;
2581 }
2582 break;
2583 }
2584 case ISD::INSERT_SUBVECTOR: {
2585 // If we know the element index, demand any elements from the subvector and
2586 // the remainder from the src its inserted into, otherwise demand them all.
2587 SDValue Src = Op.getOperand(0);
2588 SDValue Sub = Op.getOperand(1);
2589 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2590 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2591 if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
2592 Known.One.setAllBits();
2593 Known.Zero.setAllBits();
2594 uint64_t Idx = SubIdx->getZExtValue();
2595 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2596 if (!!DemandedSubElts) {
2597 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2598 if (Known.isUnknown())
2599 break; // early-out.
2600 }
2601 APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
2602 APInt DemandedSrcElts = DemandedElts & ~SubMask;
2603 if (!!DemandedSrcElts) {
2604 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2605 Known.One &= Known2.One;
2606 Known.Zero &= Known2.Zero;
2607 }
2608 } else {
2609 Known = computeKnownBits(Sub, Depth + 1);
2610 if (Known.isUnknown())
2611 break; // early-out.
2612 Known2 = computeKnownBits(Src, Depth + 1);
2613 Known.One &= Known2.One;
2614 Known.Zero &= Known2.Zero;
2615 }
2616 break;
2617 }
2618 case ISD::EXTRACT_SUBVECTOR: {
2619 // If we know the element index, just demand that subvector elements,
2620 // otherwise demand them all.
2621 SDValue Src = Op.getOperand(0);
2622 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2623 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2624 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2625 // Offset the demanded elts by the subvector index.
2626 uint64_t Idx = SubIdx->getZExtValue();
2627 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2628 Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
2629 } else {
2630 Known = computeKnownBits(Src, Depth + 1);
2631 }
2632 break;
2633 }
2634 case ISD::SCALAR_TO_VECTOR: {
2635 // We know about scalar_to_vector as much as we know about it source,
2636 // which becomes the first element of otherwise unknown vector.
2637 if (DemandedElts != 1)
2638 break;
2639
2640 SDValue N0 = Op.getOperand(0);
2641 Known = computeKnownBits(N0, Depth + 1);
2642 if (N0.getValueSizeInBits() != BitWidth)
2643 Known = Known.trunc(BitWidth);
2644
2645 break;
2646 }
2647 case ISD::BITCAST: {
2648 SDValue N0 = Op.getOperand(0);
2649 EVT SubVT = N0.getValueType();
2650 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2651
2652 // Ignore bitcasts from unsupported types.
2653 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
2654 break;
2655
2656 // Fast handling of 'identity' bitcasts.
2657 if (BitWidth == SubBitWidth) {
2658 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2659 break;
2660 }
2661
2662 bool IsLE = getDataLayout().isLittleEndian();
2663
2664 // Bitcast 'small element' vector to 'large element' scalar/vector.
2665 if ((BitWidth % SubBitWidth) == 0) {
2666 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
2667
2668 // Collect known bits for the (larger) output by collecting the known
2669 // bits from each set of sub elements and shift these into place.
2670 // We need to separately call computeKnownBits for each set of
2671 // sub elements as the knownbits for each is likely to be different.
2672 unsigned SubScale = BitWidth / SubBitWidth;
2673 APInt SubDemandedElts(NumElts * SubScale, 0);
2674 for (unsigned i = 0; i != NumElts; ++i)
2675 if (DemandedElts[i])
2676 SubDemandedElts.setBit(i * SubScale);
2677
2678 for (unsigned i = 0; i != SubScale; ++i) {
2679 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2680 Depth + 1);
2681 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
2682 Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
2683 Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
2684 }
2685 }
2686
2687 // Bitcast 'large element' scalar/vector to 'small element' vector.
2688 if ((SubBitWidth % BitWidth) == 0) {
2689 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
2690
2691 // Collect known bits for the (smaller) output by collecting the known
2692 // bits from the overlapping larger input elements and extracting the
2693 // sub sections we actually care about.
2694 unsigned SubScale = SubBitWidth / BitWidth;
2695 APInt SubDemandedElts(NumElts / SubScale, 0);
2696 for (unsigned i = 0; i != NumElts; ++i)
2697 if (DemandedElts[i])
2698 SubDemandedElts.setBit(i / SubScale);
2699
2700 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2701
2702 Known.Zero.setAllBits(); Known.One.setAllBits();
2703 for (unsigned i = 0; i != NumElts; ++i)
2704 if (DemandedElts[i]) {
2705 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
2706 unsigned Offset = (Shifts % SubScale) * BitWidth;
2707 Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2708 Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2709 // If we don't know any bits, early out.
2710 if (Known.isUnknown())
2711 break;
2712 }
2713 }
2714 break;
2715 }
2716 case ISD::AND:
2717 // If either the LHS or the RHS are Zero, the result is zero.
2718 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2719 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2720
2721 // Output known-1 bits are only known if set in both the LHS & RHS.
2722 Known.One &= Known2.One;
2723 // Output known-0 are known to be clear if zero in either the LHS | RHS.
2724 Known.Zero |= Known2.Zero;
2725 break;
2726 case ISD::OR:
2727 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2728 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2729
2730 // Output known-0 bits are only known if clear in both the LHS & RHS.
2731 Known.Zero &= Known2.Zero;
2732 // Output known-1 are known to be set if set in either the LHS | RHS.
2733 Known.One |= Known2.One;
2734 break;
2735 case ISD::XOR: {
2736 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2737 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2738
2739 // Output known-0 bits are known if clear or set in both the LHS & RHS.
2740 APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
2741 // Output known-1 are known to be set if set in only one of the LHS, RHS.
2742 Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
2743 Known.Zero = KnownZeroOut;
2744 break;
2745 }
2746 case ISD::MUL: {
2747 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2748 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2749
2750 // If low bits are zero in either operand, output low known-0 bits.
2751 // Also compute a conservative estimate for high known-0 bits.
2752 // More trickiness is possible, but this is sufficient for the
2753 // interesting case of alignment computation.
2754 unsigned TrailZ = Known.countMinTrailingZeros() +
2755 Known2.countMinTrailingZeros();
2756 unsigned LeadZ = std::max(Known.countMinLeadingZeros() +
2757 Known2.countMinLeadingZeros(),
2758 BitWidth) - BitWidth;
2759
2760 Known.resetAll();
2761 Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
2762 Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
2763 break;
2764 }
2765 case ISD::UDIV: {
2766 // For the purposes of computing leading zeros we can conservatively
2767 // treat a udiv as a logical right shift by the power of 2 known to
2768 // be less than the denominator.
2769 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2770 unsigned LeadZ = Known2.countMinLeadingZeros();
2771
2772 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2773 unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
2774 if (RHSMaxLeadingZeros != BitWidth)
2775 LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
2776
2777 Known.Zero.setHighBits(LeadZ);
2778 break;
2779 }
2780 case ISD::SELECT:
2781 case ISD::VSELECT:
2782 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2783 // If we don't know any bits, early out.
2784 if (Known.isUnknown())
2785 break;
2786 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
2787
2788 // Only known if known in both the LHS and RHS.
2789 Known.One &= Known2.One;
2790 Known.Zero &= Known2.Zero;
2791 break;
2792 case ISD::SELECT_CC:
2793 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
2794 // If we don't know any bits, early out.
2795 if (Known.isUnknown())
2796 break;
2797 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2798
2799 // Only known if known in both the LHS and RHS.
2800 Known.One &= Known2.One;
2801 Known.Zero &= Known2.Zero;
2802 break;
2803 case ISD::SMULO:
2804 case ISD::UMULO:
2805 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
2806 if (Op.getResNo() != 1)
2807 break;
2808 // The boolean result conforms to getBooleanContents.
2809 // If we know the result of a setcc has the top bits zero, use this info.
2810 // We know that we have an integer-based boolean since these operations
2811 // are only available for integer.
2812 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2813 TargetLowering::ZeroOrOneBooleanContent &&
2814 BitWidth > 1)
2815 Known.Zero.setBitsFrom(1);
2816 break;
2817 case ISD::SETCC:
2818 // If we know the result of a setcc has the top bits zero, use this info.
2819 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2820 TargetLowering::ZeroOrOneBooleanContent &&
2821 BitWidth > 1)
2822 Known.Zero.setBitsFrom(1);
2823 break;
2824 case ISD::SHL:
2825 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2826 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2827 unsigned Shift = ShAmt->getZExtValue();
2828 Known.Zero <<= Shift;
2829 Known.One <<= Shift;
2830 // Low bits are known zero.
2831 Known.Zero.setLowBits(Shift);
2832 }
2833 break;
2834 case ISD::SRL:
2835 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2836 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2837 unsigned Shift = ShAmt->getZExtValue();
2838 Known.Zero.lshrInPlace(Shift);
2839 Known.One.lshrInPlace(Shift);
2840 // High bits are known zero.
2841 Known.Zero.setHighBits(Shift);
2842 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) {
2843 // If the shift amount is a vector of constants see if we can bound
2844 // the number of upper zero bits.
2845 unsigned ShiftAmountMin = BitWidth;
2846 for (unsigned i = 0; i != BV->getNumOperands(); ++i) {
2847 if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) {
2848 const APInt &ShAmt = C->getAPIntValue();
2849 if (ShAmt.ult(BitWidth)) {
2850 ShiftAmountMin = std::min<unsigned>(ShiftAmountMin,
2851 ShAmt.getZExtValue());
2852 continue;
2853 }
2854 }
2855 // Don't know anything.
2856 ShiftAmountMin = 0;
2857 break;
2858 }
2859
2860 Known.Zero.setHighBits(ShiftAmountMin);
2861 }
2862 break;
2863 case ISD::SRA:
2864 if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2865 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2866 unsigned Shift = ShAmt->getZExtValue();
2867 // Sign extend known zero/one bit (else is unknown).
2868 Known.Zero.ashrInPlace(Shift);
2869 Known.One.ashrInPlace(Shift);
2870 }
2871 break;
2872 case ISD::FSHL:
2873 case ISD::FSHR:
2874 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
2875 unsigned Amt = C->getAPIntValue().urem(BitWidth);
2876
2877 // For fshl, 0-shift returns the 1st arg.
2878 // For fshr, 0-shift returns the 2nd arg.
2879 if (Amt == 0) {
2880 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
2881 DemandedElts, Depth + 1);
2882 break;
2883 }
2884
2885 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
2886 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
2887 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2888 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2889 if (Opcode == ISD::FSHL) {
2890 Known.One <<= Amt;
2891 Known.Zero <<= Amt;
2892 Known2.One.lshrInPlace(BitWidth - Amt);
2893 Known2.Zero.lshrInPlace(BitWidth - Amt);
2894 } else {
2895 Known.One <<= BitWidth - Amt;
2896 Known.Zero <<= BitWidth - Amt;
2897 Known2.One.lshrInPlace(Amt);
2898 Known2.Zero.lshrInPlace(Amt);
2899 }
2900 Known.One |= Known2.One;
2901 Known.Zero |= Known2.Zero;
2902 }
2903 break;
2904 case ISD::SIGN_EXTEND_INREG: {
2905 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2906 unsigned EBits = EVT.getScalarSizeInBits();
2907
2908 // Sign extension. Compute the demanded bits in the result that are not
2909 // present in the input.
2910 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2911
2912 APInt InSignMask = APInt::getSignMask(EBits);
2913 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2914
2915 // If the sign extended bits are demanded, we know that the sign
2916 // bit is demanded.
2917 InSignMask = InSignMask.zext(BitWidth);
2918 if (NewBits.getBoolValue())
2919 InputDemandedBits |= InSignMask;
2920
2921 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2922 Known.One &= InputDemandedBits;
2923 Known.Zero &= InputDemandedBits;
2924
2925 // If the sign bit of the input is known set or clear, then we know the
2926 // top bits of the result.
2927 if (Known.Zero.intersects(InSignMask)) { // Input sign bit known clear
2928 Known.Zero |= NewBits;
2929 Known.One &= ~NewBits;
2930 } else if (Known.One.intersects(InSignMask)) { // Input sign bit known set
2931 Known.One |= NewBits;
2932 Known.Zero &= ~NewBits;
2933 } else { // Input sign bit unknown
2934 Known.Zero &= ~NewBits;
2935 Known.One &= ~NewBits;
2936 }
2937 break;
2938 }
2939 case ISD::CTTZ:
2940 case ISD::CTTZ_ZERO_UNDEF: {
2941 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2942 // If we have a known 1, its position is our upper bound.
2943 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2944 unsigned LowBits = Log2_32(PossibleTZ) + 1;
2945 Known.Zero.setBitsFrom(LowBits);
2946 break;
2947 }
2948 case ISD::CTLZ:
2949 case ISD::CTLZ_ZERO_UNDEF: {
2950 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2951 // If we have a known 1, its position is our upper bound.
2952 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2953 unsigned LowBits = Log2_32(PossibleLZ) + 1;
2954 Known.Zero.setBitsFrom(LowBits);
2955 break;
2956 }
2957 case ISD::CTPOP: {
2958 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2959 // If we know some of the bits are zero, they can't be one.
2960 unsigned PossibleOnes = Known2.countMaxPopulation();
2961 Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
2962 break;
2963 }
2964 case ISD::LOAD: {
2965 LoadSDNode *LD = cast<LoadSDNode>(Op);
2966 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
2967 if (ISD::isNON_EXTLoad(LD) && Cst) {
2968 // Determine any common known bits from the loaded constant pool value.
2969 Type *CstTy = Cst->getType();
2970 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
2971 // If its a vector splat, then we can (quickly) reuse the scalar path.
2972 // NOTE: We assume all elements match and none are UNDEF.
2973 if (CstTy->isVectorTy()) {
2974 if (const Constant *Splat = Cst->getSplatValue()) {
2975 Cst = Splat;
2976 CstTy = Cst->getType();
2977 }
2978 }
2979 // TODO - do we need to handle different bitwidths?
2980 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
2981 // Iterate across all vector elements finding common known bits.
2982 Known.One.setAllBits();
2983 Known.Zero.setAllBits();
2984 for (unsigned i = 0; i != NumElts; ++i) {
2985 if (!DemandedElts[i])
2986 continue;
2987 if (Constant *Elt = Cst->getAggregateElement(i)) {
2988 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
2989 const APInt &Value = CInt->getValue();
2990 Known.One &= Value;
2991 Known.Zero &= ~Value;
2992 continue;
2993 }
2994 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
2995 APInt Value = CFP->getValueAPF().bitcastToAPInt();
2996 Known.One &= Value;
2997 Known.Zero &= ~Value;
2998 continue;
2999 }
3000 }
3001 Known.One.clearAllBits();
3002 Known.Zero.clearAllBits();
3003 break;
3004 }
3005 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3006 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3007 const APInt &Value = CInt->getValue();
3008 Known.One = Value;
3009 Known.Zero = ~Value;
3010 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3011 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3012 Known.One = Value;
3013 Known.Zero = ~Value;
3014 }
3015 }
3016 }
3017 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3018 // If this is a ZEXTLoad and we are looking at the loaded value.
3019 EVT VT = LD->getMemoryVT();
3020 unsigned MemBits = VT.getScalarSizeInBits();
3021 Known.Zero.setBitsFrom(MemBits);
3022 } else if (const MDNode *Ranges = LD->getRanges()) {
3023 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
3024 computeKnownBitsFromRangeMetadata(*Ranges, Known);
3025 }
3026 break;
3027 }
3028 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3029 EVT InVT = Op.getOperand(0).getValueType();
3030 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3031 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3032 Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
3033 break;
3034 }
3035 case ISD::ZERO_EXTEND: {
3036 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3037 Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
3038 break;
3039 }
3040 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3041 EVT InVT = Op.getOperand(0).getValueType();
3042 APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3043 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3044 // If the sign bit is known to be zero or one, then sext will extend
3045 // it to the top bits, else it will just zext.
3046 Known = Known.sext(BitWidth);
3047 break;
3048 }
3049 case ISD::SIGN_EXTEND: {
3050 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3051 // If the sign bit is known to be zero or one, then sext will extend
3052 // it to the top bits, else it will just zext.
3053 Known = Known.sext(BitWidth);
3054 break;
3055 }
3056 case ISD::ANY_EXTEND: {
3057 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3058 Known = Known.zext(BitWidth, false /* ExtendedBitsAreKnownZero */);
3059 break;
3060 }
3061 case ISD::TRUNCATE: {
3062 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3063 Known = Known.trunc(BitWidth);
3064 break;
3065 }
3066 case ISD::AssertZext: {
3067 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3068 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3069 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3070 Known.Zero |= (~InMask);
3071 Known.One &= (~Known.Zero);
3072 break;
3073 }
3074 case ISD::FGETSIGN:
3075 // All bits are zero except the low bit.
3076 Known.Zero.setBitsFrom(1);
3077 break;
3078 case ISD::USUBO:
3079 case ISD::SSUBO:
3080 if (Op.getResNo() == 1) {
3081 // If we know the result of a setcc has the top bits zero, use this info.
3082 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3083 TargetLowering::ZeroOrOneBooleanContent &&
3084 BitWidth > 1)
3085 Known.Zero.setBitsFrom(1);
3086 break;
3087 }
3088 LLVM_FALLTHROUGH;
3089 case ISD::SUB:
3090 case ISD::SUBC: {
3091 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3092 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3093 Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3094 Known, Known2);
3095 break;
3096 }
3097 case ISD::UADDO:
3098 case ISD::SADDO:
3099 case ISD::ADDCARRY:
3100 if (Op.getResNo() == 1) {
3101 // If we know the result of a setcc has the top bits zero, use this info.
3102 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3103 TargetLowering::ZeroOrOneBooleanContent &&
3104 BitWidth > 1)
3105 Known.Zero.setBitsFrom(1);
3106 break;
3107 }
3108 LLVM_FALLTHROUGH;
3109 case ISD::ADD:
3110 case ISD::ADDC:
3111 case ISD::ADDE: {
3112 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3113
3114 // With ADDE and ADDCARRY, a carry bit may be added in.
3115 KnownBits Carry(1);
3116 if (Opcode == ISD::ADDE)
3117 // Can't track carry from glue, set carry to unknown.
3118 Carry.resetAll();
3119 else if (Opcode == ISD::ADDCARRY)
3120 // TODO: Compute known bits for the carry operand. Not sure if it is worth
3121 // the trouble (how often will we find a known carry bit). And I haven't
3122 // tested this very much yet, but something like this might work:
3123 // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3124 // Carry = Carry.zextOrTrunc(1, false);
3125 Carry.resetAll();
3126 else
3127 Carry.setAllZero();
3128
3129 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3130 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3131 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3132 break;
3133 }
3134 case ISD::SREM:
3135 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3136 const APInt &RA = Rem->getAPIntValue().abs();
3137 if (RA.isPowerOf2()) {
3138 APInt LowBits = RA - 1;
3139 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3140
3141 // The low bits of the first operand are unchanged by the srem.
3142 Known.Zero = Known2.Zero & LowBits;
3143 Known.One = Known2.One & LowBits;
3144
3145 // If the first operand is non-negative or has all low bits zero, then
3146 // the upper bits are all zero.
3147 if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits))
3148 Known.Zero |= ~LowBits;
3149
3150 // If the first operand is negative and not all low bits are zero, then
3151 // the upper bits are all one.
3152 if (Known2.One[BitWidth-1] && ((Known2.One & LowBits) != 0))
3153 Known.One |= ~LowBits;
3154 assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
3155 }
3156 }
3157 break;
3158 case ISD::UREM: {
3159 if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3160 const APInt &RA = Rem->getAPIntValue();
3161 if (RA.isPowerOf2()) {
3162 APInt LowBits = (RA - 1);
3163 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3164
3165 // The upper bits are all zero, the lower ones are unchanged.
3166 Known.Zero = Known2.Zero | ~LowBits;
3167 Known.One = Known2.One & LowBits;
3168 break;
3169 }
3170 }
3171
3172 // Since the result is less than or equal to either operand, any leading
3173 // zero bits in either operand must also exist in the result.
3174 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3175 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3176
3177 uint32_t Leaders =
3178 std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
3179 Known.resetAll();
3180 Known.Zero.setHighBits(Leaders);
3181 break;
3182 }
3183 case ISD::EXTRACT_ELEMENT: {
3184 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3185 const unsigned Index = Op.getConstantOperandVal(1);
3186 const unsigned EltBitWidth = Op.getValueSizeInBits();
3187
3188 // Remove low part of known bits mask
3189 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3190 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3191
3192 // Remove high part of known bit mask
3193 Known = Known.trunc(EltBitWidth);
3194 break;
3195 }
3196 case ISD::EXTRACT_VECTOR_ELT: {
3197 SDValue InVec = Op.getOperand(0);
3198 SDValue EltNo = Op.getOperand(1);
3199 EVT VecVT = InVec.getValueType();
3200 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3201 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3202 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3203 // anything about the extended bits.
3204 if (BitWidth > EltBitWidth)
3205 Known = Known.trunc(EltBitWidth);
3206 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3207 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) {
3208 // If we know the element index, just demand that vector element.
3209 unsigned Idx = ConstEltNo->getZExtValue();
3210 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
3211 Known = computeKnownBits(InVec, DemandedElt, Depth + 1);
3212 } else {
3213 // Unknown element index, so ignore DemandedElts and demand them all.
3214 Known = computeKnownBits(InVec, Depth + 1);
3215 }
3216 if (BitWidth > EltBitWidth)
3217 Known = Known.zext(BitWidth, false /* => any extend */);
3218 break;
3219 }
3220 case ISD::INSERT_VECTOR_ELT: {
3221 SDValue InVec = Op.getOperand(0);
3222 SDValue InVal = Op.getOperand(1);
3223 SDValue EltNo = Op.getOperand(2);
3224
3225 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3226 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3227 // If we know the element index, split the demand between the
3228 // source vector and the inserted element.
3229 Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
3230 unsigned EltIdx = CEltNo->getZExtValue();
3231
3232 // If we demand the inserted element then add its common known bits.
3233 if (DemandedElts[EltIdx]) {
3234 Known2 = computeKnownBits(InVal, Depth + 1);
3235 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3236 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3237 }
3238
3239 // If we demand the source vector then add its common known bits, ensuring
3240 // that we don't demand the inserted element.
3241 APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
3242 if (!!VectorElts) {
3243 Known2 = computeKnownBits(InVec, VectorElts, Depth + 1);
3244 Known.One &= Known2.One;
3245 Known.Zero &= Known2.Zero;
3246 }
3247 } else {
3248 // Unknown element index, so ignore DemandedElts and demand them all.
3249 Known = computeKnownBits(InVec, Depth + 1);
3250 Known2 = computeKnownBits(InVal, Depth + 1);
3251 Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3252 Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3253 }
3254 break;
3255 }
3256 case ISD::BITREVERSE: {
3257 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3258 Known.Zero = Known2.Zero.reverseBits();
3259 Known.One = Known2.One.reverseBits();
3260 break;
3261 }
3262 case ISD::BSWAP: {
3263 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3264 Known.Zero = Known2.Zero.byteSwap();
3265 Known.One = Known2.One.byteSwap();
3266 break;
3267 }
3268 case ISD::ABS: {
3269 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3270
3271 // If the source's MSB is zero then we know the rest of the bits already.
3272 if (Known2.isNonNegative()) {
3273 Known.Zero = Known2.Zero;
3274 Known.One = Known2.One;
3275 break;
3276 }
3277
3278 // We only know that the absolute values's MSB will be zero iff there is
3279 // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
3280 Known2.One.clearSignBit();
3281 if (Known2.One.getBoolValue()) {
3282 Known.Zero = APInt::getSignMask(BitWidth);
3283 break;
3284 }
3285 break;
3286 }
3287 case ISD::UMIN: {
3288 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3289 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3290
3291 // UMIN - we know that the result will have the maximum of the
3292 // known zero leading bits of the inputs.
3293 unsigned LeadZero = Known.countMinLeadingZeros();
3294 LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
3295
3296 Known.Zero &= Known2.Zero;
3297 Known.One &= Known2.One;
3298 Known.Zero.setHighBits(LeadZero);
3299 break;
3300 }
3301 case ISD::UMAX: {
3302 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3303 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3304
3305 // UMAX - we know that the result will have the maximum of the
3306 // known one leading bits of the inputs.
3307 unsigned LeadOne = Known.countMinLeadingOnes();
3308 LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
3309
3310 Known.Zero &= Known2.Zero;
3311 Known.One &= Known2.One;
3312 Known.One.setHighBits(LeadOne);
3313 break;
3314 }
3315 case ISD::SMIN:
3316 case ISD::SMAX: {
3317 // If we have a clamp pattern, we know that the number of sign bits will be
3318 // the minimum of the clamp min/max range.
3319 bool IsMax = (Opcode == ISD::SMAX);
3320 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3321 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3322 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3323 CstHigh =
3324 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3325 if (CstLow && CstHigh) {
3326 if (!IsMax)
3327 std::swap(CstLow, CstHigh);
3328
3329 const APInt &ValueLow = CstLow->getAPIntValue();
3330 const APInt &ValueHigh = CstHigh->getAPIntValue();
3331 if (ValueLow.sle(ValueHigh)) {
3332 unsigned LowSignBits = ValueLow.getNumSignBits();
3333 unsigned HighSignBits = ValueHigh.getNumSignBits();
3334 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3335 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3336 Known.One.setHighBits(MinSignBits);
3337 break;
3338 }
3339 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3340 Known.Zero.setHighBits(MinSignBits);
3341 break;
3342 }
3343 }
3344 }
3345
3346 // Fallback - just get the shared known bits of the operands.
3347 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3348 if (Known.isUnknown()) break; // Early-out
3349 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3350 Known.Zero &= Known2.Zero;
3351 Known.One &= Known2.One;
3352 break;
3353 }
3354 case ISD::FrameIndex:
3355 case ISD::TargetFrameIndex:
3356 TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth);
3357 break;
3358
3359 default:
3360 if (Opcode < ISD::BUILTIN_OP_END)
3361 break;
3362 LLVM_FALLTHROUGH;
3363 case ISD::INTRINSIC_WO_CHAIN:
3364 case ISD::INTRINSIC_W_CHAIN:
3365 case ISD::INTRINSIC_VOID:
3366 // Allow the target to implement this method for its nodes.
3367 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3368 break;
3369 }
3370
3371 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
3372 return Known;
3373}
3374
3375SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3376 SDValue N1) const {
3377 // X + 0 never overflow
3378 if (isNullConstant(N1))
3379 return OFK_Never;
3380
3381 KnownBits N1Known = computeKnownBits(N1);
3382 if (N1Known.Zero.getBoolValue()) {
3383 KnownBits N0Known = computeKnownBits(N0);
3384
3385 bool overflow;
3386 (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
3387 if (!overflow)
3388 return OFK_Never;
3389 }
3390
3391 // mulhi + 1 never overflow
3392 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
3393 (~N1Known.Zero & 0x01) == ~N1Known.Zero)
3394 return OFK_Never;
3395
3396 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
3397 KnownBits N0Known = computeKnownBits(N0);
3398
3399 if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
3400 return OFK_Never;
3401 }
3402
3403 return OFK_Sometime;
3404}
3405
3406bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3407 EVT OpVT = Val.getValueType();
3408 unsigned BitWidth = OpVT.getScalarSizeInBits();
3409
3410 // Is the constant a known power of 2?
3411 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3412 return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3413
3414 // A left-shift of a constant one will have exactly one bit set because
3415 // shifting the bit off the end is undefined.
3416 if (Val.getOpcode() == ISD::SHL) {
3417 auto *C = isConstOrConstSplat(Val.getOperand(0));
3418 if (C && C->getAPIntValue() == 1)
3419 return true;
3420 }
3421
3422 // Similarly, a logical right-shift of a constant sign-bit will have exactly
3423 // one bit set.
3424 if (Val.getOpcode() == ISD::SRL) {
3425 auto *C = isConstOrConstSplat(Val.getOperand(0));
3426 if (C && C->getAPIntValue().isSignMask())
3427 return true;
3428 }
3429
3430 // Are all operands of a build vector constant powers of two?
3431 if (Val.getOpcode() == ISD::BUILD_VECTOR)
3432 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
3433 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3434 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3435 return false;
3436 }))
3437 return true;
3438
3439 // More could be done here, though the above checks are enough
3440 // to handle some common cases.
3441
3442 // Fall back to computeKnownBits to catch other known cases.
3443 KnownBits Known = computeKnownBits(Val);
3444 return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
3445}
3446
3447unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3448 EVT VT = Op.getValueType();
3449 APInt DemandedElts = VT.isVector()
3450 ? APInt::getAllOnesValue(VT.getVectorNumElements())
3451 : APInt(1, 1);
3452 return ComputeNumSignBits(Op, DemandedElts, Depth);
3453}
3454
3455unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3456 unsigned Depth) const {
3457 EVT VT = Op.getValueType();
3458 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
3459 unsigned VTBits = VT.getScalarSizeInBits();
3460 unsigned NumElts = DemandedElts.getBitWidth();
3461 unsigned Tmp, Tmp2;
3462 unsigned FirstAnswer = 1;
3463
3464 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3465 const APInt &Val = C->getAPIntValue();
3466 return Val.getNumSignBits();
3467 }
3468
3469 if (Depth == 6)
3470 return 1; // Limit search depth.
3471
3472 if (!DemandedElts)
3473 return 1; // No demanded elts, better to assume we don't know anything.
3474
3475 unsigned Opcode = Op.getOpcode();
3476 switch (Opcode) {
3477 default: break;
3478 case ISD::AssertSext:
3479 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3480 return VTBits-Tmp+1;
3481 case ISD::AssertZext:
3482 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3483 return VTBits-Tmp;
3484
3485 case ISD::BUILD_VECTOR:
3486 Tmp = VTBits;
3487 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
3488 if (!DemandedElts[i])
3489 continue;
3490
3491 SDValue SrcOp = Op.getOperand(i);
3492 Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
3493
3494 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3495 if (SrcOp.getValueSizeInBits() != VTBits) {
3496 assert(SrcOp.getValueSizeInBits() > VTBits &&
3497 "Expected BUILD_VECTOR implicit truncation");
3498 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3499 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
3500 }
3501 Tmp = std::min(Tmp, Tmp2);
3502 }
3503 return Tmp;
3504
3505 case ISD::VECTOR_SHUFFLE: {
3506 // Collect the minimum number of sign bits that are shared by every vector
3507 // element referenced by the shuffle.
3508 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3509 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3510 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3511 for (unsigned i = 0; i != NumElts; ++i) {
3512 int M = SVN->getMaskElt(i);
3513 if (!DemandedElts[i])
3514 continue;
3515 // For UNDEF elements, we don't know anything about the common state of
3516 // the shuffle result.
3517 if (M < 0)
3518 return 1;
3519 if ((unsigned)M < NumElts)
3520 DemandedLHS.setBit((unsigned)M % NumElts);
3521 else
3522 DemandedRHS.setBit((unsigned)M % NumElts);
3523 }
3524 Tmp = std::numeric_limits<unsigned>::max();
3525 if (!!DemandedLHS)
3526 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3527 if (!!DemandedRHS) {
3528 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3529 Tmp = std::min(Tmp, Tmp2);
3530 }
3531 // If we don't know anything, early out and try computeKnownBits fall-back.
3532 if (Tmp == 1)
3533 break;
3534 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3535 return Tmp;
3536 }
3537
3538 case ISD::BITCAST: {
3539 SDValue N0 = Op.getOperand(0);
3540 EVT SrcVT = N0.getValueType();
3541 unsigned SrcBits = SrcVT.getScalarSizeInBits();
3542
3543 // Ignore bitcasts from unsupported types..
3544 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
3545 break;
3546
3547 // Fast handling of 'identity' bitcasts.
3548 if (VTBits == SrcBits)
3549 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3550
3551 bool IsLE = getDataLayout().isLittleEndian();
3552
3553 // Bitcast 'large element' scalar/vector to 'small element' vector.
3554 if ((SrcBits % VTBits) == 0) {
3555 assert(VT.isVector() && "Expected bitcast to vector");
3556
3557 unsigned Scale = SrcBits / VTBits;
3558 APInt SrcDemandedElts(NumElts / Scale, 0);
3559 for (unsigned i = 0; i != NumElts; ++i)
3560 if (DemandedElts[i])
3561 SrcDemandedElts.setBit(i / Scale);
3562
3563 // Fast case - sign splat can be simply split across the small elements.
3564 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3565 if (Tmp == SrcBits)
3566 return VTBits;
3567
3568 // Slow case - determine how far the sign extends into each sub-element.
3569 Tmp2 = VTBits;
3570 for (unsigned i = 0; i != NumElts; ++i)
3571 if (DemandedElts[i]) {
3572 unsigned SubOffset = i % Scale;
3573 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
3574 SubOffset = SubOffset * VTBits;
3575 if (Tmp <= SubOffset)
3576 return 1;
3577 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3578 }
3579 return Tmp2;
3580 }
3581 break;
3582 }
3583
3584 case ISD::SIGN_EXTEND:
3585 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3586 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3587 case ISD::SIGN_EXTEND_INREG:
3588 // Max of the input and what this extends.
3589 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3590 Tmp = VTBits-Tmp+1;
3591 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3592 return std::max(Tmp, Tmp2);
3593 case ISD::SIGN_EXTEND_VECTOR_INREG: {
3594 SDValue Src = Op.getOperand(0);
3595 EVT SrcVT = Src.getValueType();
3596 APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3597 Tmp = VTBits - SrcVT.getScalarSizeInBits();
3598 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3599 }
3600
3601 case ISD::SRA:
3602 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3603 // SRA X, C -> adds C sign bits.
3604 if (ConstantSDNode *C =
3605 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3606 APInt ShiftVal = C->getAPIntValue();
3607 ShiftVal += Tmp;
3608 Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
3609 }
3610 return Tmp;
3611 case ISD::SHL:
3612 if (ConstantSDNode *C =
3613 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3614 // shl destroys sign bits.
3615 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3616 if (C->getAPIntValue().uge(VTBits) || // Bad shift.
3617 C->getAPIntValue().uge(Tmp)) break; // Shifted all sign bits out.
3618 return Tmp - C->getZExtValue();
3619 }
3620 break;
3621 case ISD::AND:
3622 case ISD::OR:
3623 case ISD::XOR: // NOT is handled here.
3624 // Logical binary ops preserve the number of sign bits at the worst.
3625 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3626 if (Tmp != 1) {
3627 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3628 FirstAnswer = std::min(Tmp, Tmp2);
3629 // We computed what we know about the sign bits as our first
3630 // answer. Now proceed to the generic code that uses
3631 // computeKnownBits, and pick whichever answer is better.
3632 }
3633 break;
3634
3635 case ISD::SELECT:
3636 case ISD::VSELECT:
3637 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3638 if (Tmp == 1) return 1; // Early out.
3639 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3640 return std::min(Tmp, Tmp2);
3641 case ISD::SELECT_CC:
3642 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3643 if (Tmp == 1) return 1; // Early out.
3644 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3645 return std::min(Tmp, Tmp2);
3646
3647 case ISD::SMIN:
3648 case ISD::SMAX: {
3649 // If we have a clamp pattern, we know that the number of sign bits will be
3650 // the minimum of the clamp min/max range.
3651 bool IsMax = (Opcode == ISD::SMAX);
3652 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3653 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3654 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3655 CstHigh =
3656 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3657 if (CstLow && CstHigh) {
3658 if (!IsMax)
3659 std::swap(CstLow, CstHigh);
3660 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3661 Tmp = CstLow->getAPIntValue().getNumSignBits();
3662 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3663 return std::min(Tmp, Tmp2);
3664 }
3665 }
3666
3667 // Fallback - just get the minimum number of sign bits of the operands.
3668 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3669 if (Tmp == 1)
3670 return 1; // Early out.
3671 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3672 return std::min(Tmp, Tmp2);
3673 }
3674 case ISD::UMIN:
3675 case ISD::UMAX:
3676 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3677 if (Tmp == 1)
3678 return 1; // Early out.
3679 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3680 return std::min(Tmp, Tmp2);
3681 case ISD::SADDO:
3682 case ISD::UADDO:
3683 case ISD::SSUBO:
3684 case ISD::USUBO:
3685 case ISD::SMULO:
3686 case ISD::UMULO:
3687 if (Op.getResNo() != 1)
3688 break;
3689 // The boolean result conforms to getBooleanContents. Fall through.
3690 // If setcc returns 0/-1, all bits are sign bits.
3691 // We know that we have an integer-based boolean since these operations
3692 // are only available for integer.
3693 if (TLI->getBooleanContents(VT.isVector(), false) ==
3694 TargetLowering::ZeroOrNegativeOneBooleanContent)
3695 return VTBits;
3696 break;
3697 case ISD::SETCC:
3698 // If setcc returns 0/-1, all bits are sign bits.
3699 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3700 TargetLowering::ZeroOrNegativeOneBooleanContent)
3701 return VTBits;
3702 break;
3703 case ISD::ROTL:
3704 case ISD::ROTR:
3705 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3706 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3707
3708 // Handle rotate right by N like a rotate left by 32-N.
3709 if (Opcode == ISD::ROTR)
3710 RotAmt = (VTBits - RotAmt) % VTBits;
3711
3712 // If we aren't rotating out all of the known-in sign bits, return the
3713 // number that are left. This handles rotl(sext(x), 1) for example.
3714 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3715 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
3716 }
3717 break;
3718 case ISD::ADD:
3719 case ISD::ADDC:
3720 // Add can have at most one carry bit. Thus we know that the output
3721 // is, at worst, one more bit than the inputs.
3722 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3723 if (Tmp == 1) return 1; // Early out.
3724
3725 // Special case decrementing a value (ADD X, -1):
3726 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
3727 if (CRHS->isAllOnesValue()) {
3728 KnownBits Known = computeKnownBits(Op.getOperand(0), Depth+1);
3729
3730 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3731 // sign bits set.
3732 if ((Known.Zero | 1).isAllOnesValue())
3733 return VTBits;
3734
3735 // If we are subtracting one from a positive number, there is no carry
3736 // out of the result.
3737 if (Known.isNonNegative())
3738 return Tmp;
3739 }
3740
3741 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3742 if (Tmp2 == 1) return 1;
3743 return std::min(Tmp, Tmp2)-1;
3744
3745 case ISD::SUB:
3746 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3747 if (Tmp2 == 1) return 1;
3748
3749 // Handle NEG.
3750 if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
3751 if (CLHS->isNullValue()) {
3752 KnownBits Known = computeKnownBits(Op.getOperand(1), Depth+1);
3753 // If the input is known to be 0 or 1, the output is 0/-1, which is all
3754 // sign bits set.
3755 if ((Known.Zero | 1).isAllOnesValue())
3756 return VTBits;
3757
3758 // If the input is known to be positive (the sign bit is known clear),
3759 // the output of the NEG has the same number of sign bits as the input.
3760 if (Known.isNonNegative())
3761 return Tmp2;
3762
3763 // Otherwise, we treat this like a SUB.
3764 }
3765
3766 // Sub can have at most one carry bit. Thus we know that the output
3767 // is, at worst, one more bit than the inputs.
3768 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3769 if (Tmp == 1) return 1; // Early out.
3770 return std::min(Tmp, Tmp2)-1;
3771 case ISD::TRUNCATE: {
3772 // Check if the sign bits of source go down as far as the truncated value.
3773 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3774 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3775 if (NumSrcSignBits > (NumSrcBits - VTBits))
3776 return NumSrcSignBits - (NumSrcBits - VTBits);
3777 break;
3778 }
3779 case ISD::EXTRACT_ELEMENT: {
3780 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3781 const int BitWidth = Op.getValueSizeInBits();
3782 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3783
3784 // Get reverse index (starting from 1), Op1 value indexes elements from
3785 // little end. Sign starts at big end.
3786 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3787
3788 // If the sign portion ends in our element the subtraction gives correct
3789 // result. Otherwise it gives either negative or > bitwidth result
3790 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3791 }
3792 case ISD::INSERT_VECTOR_ELT: {
3793 SDValue InVec = Op.getOperand(0);
3794 SDValue InVal = Op.getOperand(1);
3795 SDValue EltNo = Op.getOperand(2);
3796
3797 ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3798 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3799 // If we know the element index, split the demand between the
3800 // source vector and the inserted element.
3801 unsigned EltIdx = CEltNo->getZExtValue();
3802
3803 // If we demand the inserted element then get its sign bits.
3804 Tmp = std::numeric_limits<unsigned>::max();
3805 if (DemandedElts[EltIdx]) {
3806 // TODO - handle implicit truncation of inserted elements.
3807 if (InVal.getScalarValueSizeInBits() != VTBits)
3808 break;
3809 Tmp = ComputeNumSignBits(InVal, Depth + 1);
3810 }
3811
3812 // If we demand the source vector then get its sign bits, and determine
3813 // the minimum.
3814 APInt VectorElts = DemandedElts;
3815 VectorElts.clearBit(EltIdx);
3816 if (!!VectorElts) {
3817 Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
3818 Tmp = std::min(Tmp, Tmp2);
3819 }
3820 } else {
3821 // Unknown element index, so ignore DemandedElts and demand them all.
3822 Tmp = ComputeNumSignBits(InVec, Depth + 1);
3823 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3824 Tmp = std::min(Tmp, Tmp2);
3825 }
3826 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3827 return Tmp;
3828 }
3829 case ISD::EXTRACT_VECTOR_ELT: {
3830 SDValue InVec = Op.getOperand(0);
3831 SDValue EltNo = Op.getOperand(1);
3832 EVT VecVT = InVec.getValueType();
3833 const unsigned BitWidth = Op.getValueSizeInBits();
3834 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3835 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3836
3837 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3838 // anything about sign bits. But if the sizes match we can derive knowledge
3839 // about sign bits from the vector operand.
3840 if (BitWidth != EltBitWidth)
3841 break;
3842
3843 // If we know the element index, just demand that vector element, else for
3844 // an unknown element index, ignore DemandedElts and demand them all.
3845 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3846 ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3847 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3848 DemandedSrcElts =
3849 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3850
3851 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3852 }
3853 case ISD::EXTRACT_SUBVECTOR: {
3854 // If we know the element index, just demand that subvector elements,
3855 // otherwise demand them all.
3856 SDValue Src = Op.getOperand(0);
3857 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3858 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3859 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
3860 // Offset the demanded elts by the subvector index.
3861 uint64_t Idx = SubIdx->getZExtValue();
3862 APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
3863 return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
3864 }
3865 return ComputeNumSignBits(Src, Depth + 1);
3866 }
3867 case ISD::CONCAT_VECTORS: {
3868 // Determine the minimum number of sign bits across all demanded
3869 // elts of the input vectors. Early out if the result is already 1.
3870 Tmp = std::numeric_limits<unsigned>::max();
3871 EVT SubVectorVT = Op.getOperand(0).getValueType();
3872 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3873 unsigned NumSubVectors = Op.getNumOperands();
3874 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
3875 APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3876 DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3877 if (!DemandedSub)
3878 continue;
3879 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3880 Tmp = std::min(Tmp, Tmp2);
3881 }
3882 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3883 return Tmp;
3884 }
3885 case ISD::INSERT_SUBVECTOR: {
3886 // If we know the element index, demand any elements from the subvector and
3887 // the remainder from the src its inserted into, otherwise demand them all.
3888 SDValue Src = Op.getOperand(0);
3889 SDValue Sub = Op.getOperand(1);
3890 auto *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3891 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3892 if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
3893 Tmp = std::numeric_limits<unsigned>::max();
3894 uint64_t Idx = SubIdx->getZExtValue();
3895 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3896 if (!!DemandedSubElts) {
3897 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
3898 if (Tmp == 1) return 1; // early-out
3899 }
3900 APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
3901 APInt DemandedSrcElts = DemandedElts & ~SubMask;
3902 if (!!DemandedSrcElts) {
3903 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
3904 Tmp = std::min(Tmp, Tmp2);
3905 }
3906 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3907 return Tmp;
3908 }
3909
3910 // Not able to determine the index so just assume worst case.
3911 Tmp = ComputeNumSignBits(Sub, Depth + 1);
3912 if (Tmp == 1) return 1; // early-out
3913 Tmp2 = ComputeNumSignBits(Src, Depth + 1);
3914 Tmp = std::min(Tmp, Tmp2);
3915 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3916 return Tmp;
3917 }
3918 }
3919
3920 // If we are looking at the loaded value of the SDNode.
3921 if (Op.getResNo() == 0) {
3922 // Handle LOADX separately here. EXTLOAD case will fallthrough.
3923 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
3924 unsigned ExtType = LD->getExtensionType();
3925 switch (ExtType) {
3926 default: break;
3927 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
3928 Tmp = LD->getMemoryVT().getScalarSizeInBits();
3929 return VTBits - Tmp + 1;
3930 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
3931 Tmp = LD->getMemoryVT().getScalarSizeInBits();
3932 return VTBits - Tmp;
3933 case ISD::NON_EXTLOAD:
3934 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
3935 // We only need to handle vectors - computeKnownBits should handle
3936 // scalar cases.
3937 Type *CstTy = Cst->getType();
3938 if (CstTy->isVectorTy() &&
3939 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
3940 Tmp = VTBits;
3941 for (unsigned i = 0; i != NumElts; ++i) {
3942 if (!DemandedElts[i])
3943 continue;
3944 if (Constant *Elt = Cst->getAggregateElement(i)) {
3945 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3946 const APInt &Value = CInt->getValue();
3947 Tmp = std::min(Tmp, Value.getNumSignBits());
3948 continue;
3949 }
3950 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3951 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3952 Tmp = std::min(Tmp, Value.getNumSignBits());
3953 continue;
3954 }
3955 }
3956 // Unknown type. Conservatively assume no bits match sign bit.
3957 return 1;
3958 }
3959 return Tmp;
3960 }
3961 }
3962 break;
3963 }
3964 }
3965 }
3966
3967 // Allow the target to implement this method for its nodes.
3968 if (Opcode >= ISD::BUILTIN_OP_END ||
3969 Opcode == ISD::INTRINSIC_WO_CHAIN ||
3970 Opcode == ISD::INTRINSIC_W_CHAIN ||
3971 Opcode == ISD::INTRINSIC_VOID) {
3972 unsigned NumBits =
3973 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
3974 if (NumBits > 1)
3975 FirstAnswer = std::max(FirstAnswer, NumBits);
3976 }
3977
3978 // Finally, if we can prove that the top bits of the result are 0's or 1's,
3979 // use this information.
3980 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
3981
3982 APInt Mask;
3983 if (Known.isNonNegative()) { // sign bit is 0
3984 Mask = Known.Zero;
3985 } else if (Known.isNegative()) { // sign bit is 1;
3986 Mask = Known.One;
3987 } else {
3988 // Nothing known.
3989 return FirstAnswer;
3990 }
3991
3992 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
3993 // the number of identical bits in the top of the input value.
3994 Mask = ~Mask;
3995 Mask <<= Mask.getBitWidth()-VTBits;
3996 // Return # leading zeros. We use 'min' here in case Val was zero before
3997 // shifting. We don't want to return '64' as for an i32 "0".
3998 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
3999}
4000
4001bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
4002 switch (Op.getOpcode()) {
4003 case ISD::ADD:
4004 case ISD::OR:
4005 case ISD::PTRADD:
4006 if (isa<ConstantSDNode>(Op.getOperand(1)))
4007 break;
4008 LLVM_FALLTHROUGH;
4009 default:
4010 return false;
4011 }
4012
4013 if (Op.getOpcode() == ISD::OR &&
4014 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
4015 return false;
4016
4017 return true;
4018}
4019
4020bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
4021 // If we're told that NaNs won't happen, assume they won't.
4022 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
4023 return true;
4024
4025 if (Depth == 6)
4026 return false; // Limit search depth.
4027
4028 // TODO: Handle vectors.
4029 // If the value is a constant, we can obviously see if it is a NaN or not.
4030 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
4031 return !C->getValueAPF().isNaN() ||
4032 (SNaN && !C->getValueAPF().isSignaling());
4033 }
4034
4035 unsigned Opcode = Op.getOpcode();
4036 switch (Opcode) {
4037 case ISD::FADD:
4038 case ISD::FSUB:
4039 case ISD::FMUL:
4040 case ISD::FDIV:
4041 case ISD::FREM:
4042 case ISD::FSIN:
4043 case ISD::FCOS: {
4044 if (SNaN)
4045 return true;
4046 // TODO: Need isKnownNeverInfinity
4047 return false;
4048 }
4049 case ISD::FCANONICALIZE:
4050 case ISD::FEXP:
4051 case ISD::FEXP2:
4052 case ISD::FTRUNC:
4053 case ISD::FFLOOR:
4054 case ISD::FCEIL:
4055 case ISD::FROUND:
4056 case ISD::FRINT:
4057 case ISD::FNEARBYINT: {
4058 if (SNaN)
4059 return true;
4060 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4061 }
4062 case ISD::FABS:
4063 case ISD::FNEG:
4064 case ISD::FCOPYSIGN: {
4065 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4066 }
4067 case ISD::SELECT:
4068 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4069 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4070 case ISD::FP_EXTEND:
4071 case ISD::FP_ROUND: {
4072 if (SNaN)
4073 return true;
4074 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4075 }
4076 case ISD::SINT_TO_FP:
4077 case ISD::UINT_TO_FP:
4078 return true;
4079 case ISD::FMA:
4080 case ISD::FMAD: {
4081 if (SNaN)
4082 return true;
4083 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4084 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4085 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4086 }
4087 case ISD::FSQRT: // Need is known positive
4088 case ISD::FLOG:
4089 case ISD::FLOG2:
4090 case ISD::FLOG10:
4091 case ISD::FPOWI:
4092 case ISD::FPOW: {
4093 if (SNaN)
4094 return true;
4095 // TODO: Refine on operand
4096 return false;
4097 }
4098 case ISD::FMINNUM:
4099 case ISD::FMAXNUM: {
4100 // Only one needs to be known not-nan, since it will be returned if the
4101 // other ends up being one.
4102 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4103 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4104 }
4105 case ISD::FMINNUM_IEEE:
4106 case ISD::FMAXNUM_IEEE: {
4107 if (SNaN)
4108 return true;
4109 // This can return a NaN if either operand is an sNaN, or if both operands
4110 // are NaN.
4111 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4112 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4113 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4114 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4115 }
4116 case ISD::FMINIMUM:
4117 case ISD::FMAXIMUM: {
4118 // TODO: Does this quiet or return the origina NaN as-is?
4119 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4120 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4121 }
4122 case ISD::EXTRACT_VECTOR_ELT: {
4123 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4124 }
4125 default:
4126 if (Opcode >= ISD::BUILTIN_OP_END ||
4127 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4128 Opcode == ISD::INTRINSIC_W_CHAIN ||
4129 Opcode == ISD::INTRINSIC_VOID) {
4130 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4131 }
4132
4133 return false;
4134 }
4135}
4136
4137bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4138 assert(Op.getValueType().isFloatingPoint() &&
4139 "Floating point type expected");
4140
4141 // If the value is a constant, we can obviously see if it is a zero or not.
4142 // TODO: Add BuildVector support.
4143 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4144 return !C->isZero();
4145 return false;
4146}
4147
4148bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4149 assert(!Op.getValueType().isFloatingPoint() &&
4150 "Floating point types unsupported - use isKnownNeverZeroFloat");
4151
4152 // If the value is a constant, we can obviously see if it is a zero or not.
4153 if (ISD::matchUnaryPredicate(
4154 Op, [](ConstantSDNode *C) { return !C->isNullValue(); }))
4155 return true;
4156
4157 // TODO: Recognize more cases here.
4158 switch (Op.getOpcode()) {
4159 default: break;
4160 case ISD::OR:
4161 if (isKnownNeverZero(Op.getOperand(1)) ||
4162 isKnownNeverZero(Op.getOperand(0)))
4163 return true;
4164 break;
4165 }
4166
4167 return false;
4168}
4169
4170bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4171 // Check the obvious case.
4172 if (A == B) return true;
4173
4174 // For for negative and positive zero.
4175 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4176 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4177 if (CA->isZero() && CB->isZero()) return true;
4178
4179 // Otherwise they may not be equal.
4180 return false;
4181}
4182
4183// FIXME: unify with llvm::haveNoCommonBitsSet.
4184// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4185bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4186 assert(A.getValueType() == B.getValueType() &&
4187 "Values must have the same type");
4188 return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
4189}
4190
4191static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4192 ArrayRef<SDValue> Ops,
4193 SelectionDAG &DAG) {
4194 int NumOps = Ops.size();
4195 assert(NumOps != 0 && "Can't build an empty vector!");
4196 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
4197 "Incorrect element count in BUILD_VECTOR!");
4198
4199 // BUILD_VECTOR of UNDEFs is UNDEF.
4200 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4201 return DAG.getUNDEF(VT);
4202
4203 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4204 SDValue IdentitySrc;
4205 bool IsIdentity = true;
4206 for (int i = 0; i != NumOps; ++i) {
4207 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4208 Ops[i].getOperand(0).getValueType() != VT ||
4209 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
4210 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
4211 cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i) {
4212 IsIdentity = false;
4213 break;
4214 }
4215 IdentitySrc = Ops[i].getOperand(0);
4216 }
4217 if (IsIdentity)
4218 return IdentitySrc;
4219
4220 return SDValue();
4221}
4222
4223/// Try to simplify vector concatenation to an input value, undef, or build
4224/// vector.
4225static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4226 ArrayRef<SDValue> Ops,
4227 SelectionDAG &DAG) {
4228 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
4229 assert(llvm::all_of(Ops,
4230 [Ops](SDValue Op) {
4231 return Ops[0].getValueType() == Op.getValueType();
4232 }) &&
4233 "Concatenation of vectors with inconsistent value types!");
4234 assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
4235 VT.getVectorNumElements() &&
4236 "Incorrect element count in vector concatenation!");
4237
4238 if (Ops.size() == 1)
4239 return Ops[0];
4240
4241 // Concat of UNDEFs is UNDEF.
4242 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
4243 return DAG.getUNDEF(VT);
4244
4245 // Scan the operands and look for extract operations from a single source
4246 // that correspond to insertion at the same location via this concatenation:
4247 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4248 SDValue IdentitySrc;
4249 bool IsIdentity = true;
4250 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
4251 SDValue Op = Ops[i];
4252 unsigned IdentityIndex = i * Op.getValueType().getVectorNumElements();
4253 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4254 Op.getOperand(0).getValueType() != VT ||
4255 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
4256 !isa<ConstantSDNode>(Op.getOperand(1)) ||
4257 Op.getConstantOperandVal(1) != IdentityIndex) {
4258 IsIdentity = false;
4259 break;
4260 }
4261 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
4262 "Unexpected identity source vector for concat of extracts");
4263 IdentitySrc = Op.getOperand(0);
4264 }
4265 if (IsIdentity) {
4266 assert(IdentitySrc && "Failed to set source vector of extracts");
4267 return IdentitySrc;
4268 }
4269
4270 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4271 // simplified to one big BUILD_VECTOR.
4272 // FIXME: Add support for SCALAR_TO_VECTOR as well.
4273 EVT SVT = VT.getScalarType();
4274 SmallVector<SDValue, 16> Elts;
4275 for (SDValue Op : Ops) {
4276 EVT OpVT = Op.getValueType();
4277 if (Op.isUndef())
4278 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4279 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4280 Elts.append(Op->op_begin(), Op->op_end());
4281 else
4282 return SDValue();
4283 }
4284
4285 // BUILD_VECTOR requires all inputs to be of the same type, find the
4286 // maximum type and extend them all.
4287 for (SDValue Op : Elts)
4288 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
4289
4290 if (SVT.bitsGT(VT.getScalarType()))
4291 for (SDValue &Op : Elts)
4292 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4293 ? DAG.getZExtOrTrunc(Op, DL, SVT)
4294 : DAG.getSExtOrTrunc(Op, DL, SVT);
4295
4296 SDValue V = DAG.getBuildVector(VT, DL, Elts);
4297 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4298 return V;
4299}
4300
4301/// Gets or creates the specified node.
4302SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4303 FoldingSetNodeID ID;
4304 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4305 void *IP = nullptr;
4306 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4307 return SDValue(E, 0);
4308
4309 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4310 getVTList(VT));
4311 CSEMap.InsertNode(N, IP);
4312
4313 InsertNode(N);
4314 SDValue V = SDValue(N, 0);
4315 NewSDValueDbgMsg(V, "Creating new node: ", this);
4316 return V;
4317}
4318
4319SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4320 SDValue Operand, const SDNodeFlags Flags) {
4321 // Constant fold unary operations with an integer constant operand. Even
4322 // opaque constant will be folded, because the folding of unary operations
4323 // doesn't create new constants with different values. Nevertheless, the
4324 // opaque flag is preserved during folding to prevent future folding with
4325 // other constants.
4326 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4327 const APInt &Val = C->getAPIntValue();
4328 switch (Opcode) {
4329 default: break;
4330 case ISD::SIGN_EXTEND:
4331 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4332 C->isTargetOpcode(), C->isOpaque());
4333 case ISD::TRUNCATE:
4334 if (C->isOpaque())
4335 break;
4336 LLVM_FALLTHROUGH;
4337 case ISD::ANY_EXTEND:
4338 case ISD::ZERO_EXTEND:
4339 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4340 C->isTargetOpcode(), C->isOpaque());
4341 case ISD::UINT_TO_FP:
4342 case ISD::SINT_TO_FP: {
4343 APFloat apf(EVTToAPFloatSemantics(VT),
4344 APInt::getNullValue(VT.getSizeInBits()));
4345 (void)apf.convertFromAPInt(Val,
4346 Opcode==ISD::SINT_TO_FP,
4347 APFloat::rmNearestTiesToEven);
4348 return getConstantFP(apf, DL, VT);
4349 }
4350 case ISD::BITCAST:
4351 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
4352 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4353 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
4354 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4355 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
4356 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4357 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
4358 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4359 break;
4360 case ISD::ABS:
4361 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4362 C->isOpaque());
4363 case ISD::BITREVERSE:
4364 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4365 C->isOpaque());
4366 case ISD::BSWAP:
4367 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4368 C->isOpaque());
4369 case ISD::CTPOP:
4370 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4371 C->isOpaque());
4372 case ISD::CTLZ:
4373 case ISD::CTLZ_ZERO_UNDEF:
4374 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4375 C->isOpaque());
4376 case ISD::CTTZ:
4377 case ISD::CTTZ_ZERO_UNDEF:
4378 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4379 C->isOpaque());
4380 case ISD::FP16_TO_FP: {
4381 bool Ignored;
4382 APFloat FPV(APFloat::IEEEhalf(),
4383 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
4384
4385 // This can return overflow, underflow, or inexact; we don't care.
4386 // FIXME need to be more flexible about rounding mode.
4387 (void)FPV.convert(EVTToAPFloatSemantics(VT),
4388 APFloat::rmNearestTiesToEven, &Ignored);
4389 return getConstantFP(FPV, DL, VT);
4390 }
4391 }
4392 }
4393
4394 // Constant fold unary operations with a floating point constant operand.
4395 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4396 APFloat V = C->getValueAPF(); // make copy
4397 switch (Opcode) {
4398 case ISD::FNEG:
4399 V.changeSign();
4400 return getConstantFP(V, DL, VT);
4401 case ISD::FABS:
4402 V.clearSign();
4403 return getConstantFP(V, DL, VT);
4404 case ISD::FCEIL: {
4405 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4406 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4407 return getConstantFP(V, DL, VT);
4408 break;
4409 }
4410 case ISD::FTRUNC: {
4411 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4412 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4413 return getConstantFP(V, DL, VT);
4414 break;
4415 }
4416 case ISD::FFLOOR: {
4417 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4418 if (fs == APFloat::opOK || fs == APFloat::opInexact)
4419 return getConstantFP(V, DL, VT);
4420 break;
4421 }
4422 case ISD::FP_EXTEND: {
4423 bool ignored;
4424 // This can return overflow, underflow, or inexact; we don't care.
4425 // FIXME need to be more flexible about rounding mode.
4426 (void)V.convert(EVTToAPFloatSemantics(VT),
4427 APFloat::rmNearestTiesToEven, &ignored);
4428 return getConstantFP(V, DL, VT);
4429 }
4430 case ISD::FP_TO_SINT:
4431 case ISD::FP_TO_UINT: {
4432 bool ignored;
4433 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4434 // FIXME need to be more flexible about rounding mode.
4435 APFloat::opStatus s =
4436 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4437 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4438 break;
4439 return getConstant(IntVal, DL, VT);
4440 }
4441 case ISD::BITCAST:
4442 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
4443 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4444 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
4445 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4446 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
4447 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4448 break;
4449 case ISD::FP_TO_FP16: {
4450 bool Ignored;
4451 // This can return overflow, underflow, or inexact; we don't care.
4452 // FIXME need to be more flexible about rounding mode.
4453 (void)V.convert(APFloat::IEEEhalf(),
4454 APFloat::rmNearestTiesToEven, &Ignored);
4455 return getConstant(V.bitcastToAPInt(), DL, VT);
4456 }
4457 }
4458 }
4459
4460 // Constant fold unary operations with a vector integer or float operand.
4461 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
4462 if (BV->isConstant()) {
4463 switch (Opcode) {
4464 default:
4465 // FIXME: Entirely reasonable to perform folding of other unary
4466 // operations here as the need arises.
4467 break;
4468 case ISD::FNEG:
4469 case ISD::FABS:
4470 case ISD::FCEIL:
4471 case ISD::FTRUNC:
4472 case ISD::FFLOOR:
4473 case ISD::FP_EXTEND:
4474 case ISD::FP_TO_SINT:
4475 case ISD::FP_TO_UINT:
4476 case ISD::TRUNCATE:
4477 case ISD::ANY_EXTEND:
4478 case ISD::ZERO_EXTEND:
4479 case ISD::SIGN_EXTEND:
4480 case ISD::UINT_TO_FP:
4481 case ISD::SINT_TO_FP:
4482 case ISD::ABS:
4483 case ISD::BITREVERSE:
4484 case ISD::BSWAP:
4485 case ISD::CTLZ:
4486 case ISD::CTLZ_ZERO_UNDEF:
4487 case ISD::CTTZ:
4488 case ISD::CTTZ_ZERO_UNDEF:
4489 case ISD::CTPOP: {
4490 SDValue Ops = { Operand };
4491 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4492 return Fold;
4493 }
4494 }
4495 }
4496 }
4497
4498 unsigned OpOpcode = Operand.getNode()->getOpcode();
4499 switch (Opcode) {
4500 case ISD::TokenFactor:
4501 case ISD::MERGE_VALUES:
4502 case ISD::CONCAT_VECTORS:
4503 return Operand; // Factor, merge or concat of one node? No need.
4504 case ISD::BUILD_VECTOR: {
4505 // Attempt to simplify BUILD_VECTOR.
4506 SDValue Ops[] = {Operand};
4507 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4508 return V;
4509 break;
4510 }
4511 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
4512 case ISD::FP_EXTEND:
4513 assert(VT.isFloatingPoint() &&
4514 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
4515 if (Operand.getValueType() == VT) return Operand; // noop conversion.
4516 assert((!VT.isVector() ||
4517 VT.getVectorNumElements() ==
4518 Operand.getValueType().getVectorNumElements()) &&
4519 "Vector element count mismatch!");
4520 assert(Operand.getValueType().bitsLT(VT) &&
4521 "Invalid fpext node, dst < src!");
4522 if (Operand.isUndef())
4523 return getUNDEF(VT);
4524 break;
4525 case ISD::FP_TO_SINT:
4526 case ISD::FP_TO_UINT:
4527 if (Operand.isUndef())
4528 return getUNDEF(VT);
4529 break;
4530 case ISD::SINT_TO_FP:
4531 case ISD::UINT_TO_FP:
4532 // [us]itofp(undef) = 0, because the result value is bounded.
4533 if (Operand.isUndef())
4534 return getConstantFP(0.0, DL, VT);
4535 break;
4536 case ISD::SIGN_EXTEND:
4537 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4538 "Invalid SIGN_EXTEND!");
4539 assert(VT.isVector() == Operand.getValueType().isVector() &&
4540 "SIGN_EXTEND result type type should be vector iff the operand "
4541 "type is vector!");
4542 if (Operand.getValueType() == VT) return Operand; // noop extension
4543 assert((!VT.isVector() ||
4544 VT.getVectorNumElements() ==
4545 Operand.getValueType().getVectorNumElements()) &&
4546 "Vector element count mismatch!");
4547 assert(Operand.getValueType().bitsLT(VT) &&
4548 "Invalid sext node, dst < src!");
4549 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
4550 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4551 else if (OpOpcode == ISD::UNDEF)
4552 // sext(undef) = 0, because the top bits will all be the same.
4553 return getConstant(0, DL, VT);
4554 break;
4555 case ISD::ZERO_EXTEND:
4556 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4557 "Invalid ZERO_EXTEND!");
4558 assert(VT.isVector() == Operand.getValueType().isVector() &&
4559 "ZERO_EXTEND result type type should be vector iff the operand "
4560 "type is vector!");
4561 if (Operand.getValueType() == VT) return Operand; // noop extension
4562 assert((!VT.isVector() ||
4563 VT.getVectorNumElements() ==
4564 Operand.getValueType().getVectorNumElements()) &&
4565 "Vector element count mismatch!");
4566 assert(Operand.getValueType().bitsLT(VT) &&
4567 "Invalid zext node, dst < src!");
4568 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
4569 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4570 else if (OpOpcode == ISD::UNDEF)
4571 // zext(undef) = 0, because the top bits will be zero.
4572 return getConstant(0, DL, VT);
4573 break;
4574 case ISD::ANY_EXTEND:
4575 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4576 "Invalid ANY_EXTEND!");
4577 assert(VT.isVector() == Operand.getValueType().isVector() &&
4578 "ANY_EXTEND result type type should be vector iff the operand "
4579 "type is vector!");
4580 if (Operand.getValueType() == VT) return Operand; // noop extension
4581 assert((!VT.isVector() ||
4582 VT.getVectorNumElements() ==
4583 Operand.getValueType().getVectorNumElements()) &&
4584 "Vector element count mismatch!");
4585 assert(Operand.getValueType().bitsLT(VT) &&
4586 "Invalid anyext node, dst < src!");
4587
4588 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4589 OpOpcode == ISD::ANY_EXTEND)
4590 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
4591 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4592 else if (OpOpcode == ISD::UNDEF)
4593 return getUNDEF(VT);
4594
4595 // (ext (trunc x)) -> x
4596 if (OpOpcode == ISD::TRUNCATE) {
4597 SDValue OpOp = Operand.getOperand(0);
4598 if (OpOp.getValueType() == VT) {
4599 transferDbgValues(Operand, OpOp);
4600 return OpOp;
4601 }
4602 }
4603 break;
4604 case ISD::TRUNCATE:
4605 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4606 "Invalid TRUNCATE!");
4607 assert(VT.isVector() == Operand.getValueType().isVector() &&
4608 "TRUNCATE result type type should be vector iff the operand "
4609 "type is vector!");
4610 if (Operand.getValueType() == VT) return Operand; // noop truncate
4611 assert((!VT.isVector() ||
4612 VT.getVectorNumElements() ==
4613 Operand.getValueType().getVectorNumElements()) &&
4614 "Vector element count mismatch!");
4615 assert(Operand.getValueType().bitsGT(VT) &&
4616 "Invalid truncate node, src < dst!");
4617 if (OpOpcode == ISD::TRUNCATE)
4618 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4619 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
4620 OpOpcode == ISD::ANY_EXTEND) {
4621 // If the source is smaller than the dest, we still need an extend.
4622 if (Operand.getOperand(0).getValueType().getScalarType()
4623 .bitsLT(VT.getScalarType()))
4624 return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4625 if (Operand.getOperand(0).getValueType().bitsGT(VT))
4626 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4627 return Operand.getOperand(0);
4628 }
4629 if (OpOpcode == ISD::UNDEF)
4630 return getUNDEF(VT);
4631 break;
4632 case ISD::ANY_EXTEND_VECTOR_INREG:
4633 case ISD::ZERO_EXTEND_VECTOR_INREG:
4634 case ISD::SIGN_EXTEND_VECTOR_INREG:
4635 assert(VT.isVector() && "This DAG node is restricted to vector types.");
4636 assert(Operand.getValueType().bitsLE(VT) &&
4637 "The input must be the same size or smaller than the result.");
4638 assert(VT.getVectorNumElements() <
4639 Operand.getValueType().getVectorNumElements() &&
4640 "The destination vector type must have fewer lanes than the input.");
4641 break;
4642 case ISD::ABS:
4643 assert(VT.isInteger() && VT == Operand.getValueType() &&
4644 "Invalid ABS!");
4645 if (OpOpcode == ISD::UNDEF)
4646 return getUNDEF(VT);
4647 break;
4648 case ISD::BSWAP:
4649 assert(VT.isInteger() && VT == Operand.getValueType() &&
4650 "Invalid BSWAP!");
4651 assert((VT.getScalarSizeInBits() % 16 == 0) &&
4652 "BSWAP types must be a multiple of 16 bits!");
4653 if (OpOpcode == ISD::UNDEF)
4654 return getUNDEF(VT);
4655 break;
4656 case ISD::BITREVERSE:
4657 assert(VT.isInteger() && VT == Operand.getValueType() &&
4658 "Invalid BITREVERSE!");
4659 if (OpOpcode == ISD::UNDEF)
4660 return getUNDEF(VT);
4661 break;
4662 case ISD::BITCAST:
4663 // Basic sanity checking.
4664 assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
4665 "Cannot BITCAST between types of different sizes!");
4666 if (VT == Operand.getValueType()) return Operand; // noop conversion.
4667 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
4668 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
4669 if (OpOpcode == ISD::UNDEF)
4670 return getUNDEF(VT);
4671 break;
4672 case ISD::SCALAR_TO_VECTOR:
4673 assert(VT.isVector() && !Operand.getValueType().isVector() &&
4674 (VT.getVectorElementType() == Operand.getValueType() ||
4675 (VT.getVectorElementType().isInteger() &&
4676 Operand.getValueType().isInteger() &&
4677 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
4678 "Illegal SCALAR_TO_VECTOR node!");
4679 if (OpOpcode == ISD::UNDEF)
4680 return getUNDEF(VT);
4681 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
4682 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
4683 isa<ConstantSDNode>(Operand.getOperand(1)) &&
4684 Operand.getConstantOperandVal(1) == 0 &&
4685 Operand.getOperand(0).getValueType() == VT)
4686 return Operand.getOperand(0);
4687 break;
4688 case ISD::FNEG:
4689 // Negation of an unknown bag of bits is still completely undefined.
4690 if (OpOpcode == ISD::UNDEF)
4691 return getUNDEF(VT);
4692
4693 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
4694 if ((getTarget().Options.UnsafeFPMath || Flags.hasNoSignedZeros()) &&
4695 OpOpcode == ISD::FSUB)
4696 return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
4697 Operand.getOperand(0), Flags);
4698 if (OpOpcode == ISD::FNEG) // --X -> X
4699 return Operand.getOperand(0);
4700 break;
4701 case ISD::FABS:
4702 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
4703 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
4704 break;
4705 }
4706
4707 SDNode *N;
4708 SDVTList VTs = getVTList(VT);
4709 SDValue Ops[] = {Operand};
4710 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
4711 FoldingSetNodeID ID;
4712 AddNodeIDNode(ID, Opcode, VTs, Ops);
4713 void *IP = nullptr;
4714 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4715 E->intersectFlagsWith(Flags);
4716 return SDValue(E, 0);
4717 }
4718
4719 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4720 N->setFlags(Flags);
4721 createOperands(N, Ops);
4722 CSEMap.InsertNode(N, IP);
4723 } else {
4724 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4725 createOperands(N, Ops);
4726 }
4727
4728 InsertNode(N);
4729 SDValue V = SDValue(N, 0);
4730 NewSDValueDbgMsg(V, "Creating new node: ", this);
4731 return V;
4732}
4733
4734static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
4735 const APInt &C2) {
4736 switch (Opcode) {
4737 case ISD::ADD: return std::make_pair(C1 + C2, true);
4738 case ISD::SUB: return std::make_pair(C1 - C2, true);
4739 case ISD::MUL: return std::make_pair(C1 * C2, true);
4740 case ISD::AND: return std::make_pair(C1 & C2, true);
4741 case ISD::OR: return std::make_pair(C1 | C2, true);
4742 case ISD::XOR: return std::make_pair(C1 ^ C2, true);
4743 case ISD::SHL: return std::make_pair(C1 << C2, true);
4744 case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
4745 case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
4746 case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
4747 case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
4748 case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
4749 case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
4750 case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
4751 case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
4752 case ISD::SADDSAT: return std::make_pair(C1.sadd_sat(C2), true);
4753 case ISD::UADDSAT: return std::make_pair(C1.uadd_sat(C2), true);
4754 case ISD::SSUBSAT: return std::make_pair(C1.ssub_sat(C2), true);
4755 case ISD::USUBSAT: return std::make_pair(C1.usub_sat(C2), true);
4756 case ISD::UDIV:
4757 if (!C2.getBoolValue())
4758 break;
4759 return std::make_pair(C1.udiv(C2), true);
4760 case ISD::UREM:
4761 if (!C2.getBoolValue())
4762 break;
4763 return std::make_pair(C1.urem(C2), true);
4764 case ISD::SDIV:
4765 if (!C2.getBoolValue())
4766 break;
4767 return std::make_pair(C1.sdiv(C2), true);
4768 case ISD::SREM:
4769 if (!C2.getBoolValue())
4770 break;
4771 return std::make_pair(C1.srem(C2), true);
4772 }
4773 return std::make_pair(APInt(1, 0), false);
4774}
4775
4776SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4777 EVT VT, const ConstantSDNode *C1,
4778 const ConstantSDNode *C2) {
4779 if (C1->isOpaque() || C2->isOpaque())
4780 return SDValue();
4781
4782 std::pair<APInt, bool> Folded = FoldValue(Opcode, C1->getAPIntValue(),
4783 C2->getAPIntValue());
4784 if (!Folded.second)
4785 return SDValue();
4786 return getConstant(Folded.first, DL, VT);
4787}
4788
4789SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4790 const GlobalAddressSDNode *GA,
4791 const SDNode *N2) {
4792 if (GA->getOpcode() != ISD::GlobalAddress)
4793 return SDValue();
4794 if (!TLI->isOffsetFoldingLegal(GA))
4795 return SDValue();
4796 auto *C2 = dyn_cast<ConstantSDNode>(N2);
4797 if (!C2)
4798 return SDValue();
4799 int64_t Offset = C2->getSExtValue();
4800 switch (Opcode) {
4801 case ISD::ADD: break;
4802 case ISD::SUB: Offset = -uint64_t(Offset); break;
4803 default: return SDValue();
4804 }
4805 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
4806 GA->getOffset() + uint64_t(Offset));
4807}
4808
4809bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4810 switch (Opcode) {
4811 case ISD::SDIV:
4812 case ISD::UDIV:
4813 case ISD::SREM:
4814 case ISD::UREM: {
4815 // If a divisor is zero/undef or any element of a divisor vector is
4816 // zero/undef, the whole op is undef.
4817 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
4818 SDValue Divisor = Ops[1];
4819 if (Divisor.isUndef() || isNullConstant(Divisor))
4820 return true;
4821
4822 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
4823 llvm::any_of(Divisor->op_values(),
4824 [](SDValue V) { return V.isUndef() ||
4825 isNullConstant(V); });
4826 // TODO: Handle signed overflow.
4827 }
4828 // TODO: Handle oversized shifts.
4829 default:
4830 return false;
4831 }
4832}
4833
4834SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4835 EVT VT, SDNode *N1, SDNode *N2) {
4836 // If the opcode is a target-specific ISD node, there's nothing we can
4837 // do here and the operand rules may not line up with the below, so
4838 // bail early.
4839 if (Opcode >= ISD::BUILTIN_OP_END)
4840 return SDValue();
4841
4842 if (isUndef(Opcode, {SDValue(N1, 0), SDValue(N2, 0)}))
4843 return getUNDEF(VT);
4844
4845 // Handle the case of two scalars.
4846 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
4847 if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
4848 SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2);
4849 assert((!Folded || !VT.isVector()) &&
4850 "Can't fold vectors ops with scalar operands");
4851 return Folded;
4852 }
4853 }
4854
4855 // fold (add Sym, c) -> Sym+c
4856 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
4857 return FoldSymbolOffset(Opcode, VT, GA, N2);
4858 if (TLI->isCommutativeBinOp(Opcode))
4859 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
4860 return FoldSymbolOffset(Opcode, VT, GA, N1);
4861
4862 // For vectors, extract each constant element and fold them individually.
4863 // Either input may be an undef value.
4864 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
4865 if (!BV1 && !N1->isUndef())
4866 return SDValue();
4867 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
4868 if (!BV2 && !N2->isUndef())
4869 return SDValue();
4870 // If both operands are undef, that's handled the same way as scalars.
4871 if (!BV1 && !BV2)
4872 return SDValue();
4873
4874 assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) &&
4875 "Vector binop with different number of elements in operands?");
4876
4877 EVT SVT = VT.getScalarType();
4878 EVT LegalSVT = SVT;
4879 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4880 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4881 if (LegalSVT.bitsLT(SVT))
4882 return SDValue();
4883 }
4884 SmallVector<SDValue, 4> Outputs;
4885 unsigned NumOps = BV1 ? BV1->getNumOperands() : BV2->getNumOperands();
4886 for (unsigned I = 0; I != NumOps; ++I) {
4887 SDValue V1 = BV1 ? BV1->getOperand(I) : getUNDEF(SVT);
4888 SDValue V2 = BV2 ? BV2->getOperand(I) : getUNDEF(SVT);
4889 if (SVT.isInteger()) {
4890 if (V1->getValueType(0).bitsGT(SVT))
4891 V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4892 if (V2->getValueType(0).bitsGT(SVT))
4893 V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4894 }
4895
4896 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
4897 return SDValue();
4898
4899 // Fold one vector element.
4900 SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4901 if (LegalSVT != SVT)
4902 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4903
4904 // Scalar folding only succeeded if the result is a constant or UNDEF.
4905 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
4906 ScalarResult.getOpcode() != ISD::ConstantFP)
4907 return SDValue();
4908 Outputs.push_back(ScalarResult);
4909 }
4910
4911 assert(VT.getVectorNumElements() == Outputs.size() &&
4912 "Vector size mismatch!");
4913
4914 // We may have a vector type but a scalar result. Create a splat.
4915 Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4916
4917 // Build a big vector out of the scalar elements we generated.
4918 return getBuildVector(VT, SDLoc(), Outputs);
4919}
4920
4921// TODO: Merge with FoldConstantArithmetic
4922SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4923 const SDLoc &DL, EVT VT,
4924 ArrayRef<SDValue> Ops,
4925 const SDNodeFlags Flags) {
4926 // If the opcode is a target-specific ISD node, there's nothing we can
4927 // do here and the operand rules may not line up with the below, so
4928 // bail early.
4929 if (Opcode >= ISD::BUILTIN_OP_END)
4930 return SDValue();
4931
4932 if (isUndef(Opcode, Ops))
4933 return getUNDEF(VT);
4934
4935 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
4936 if (!VT.isVector())
4937 return SDValue();
4938
4939 unsigned NumElts = VT.getVectorNumElements();
4940
4941 auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
4942 return !Op.getValueType().isVector() ||
4943 Op.getValueType().getVectorNumElements() == NumElts;
4944 };
4945
4946 auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
4947 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
4948 return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
4949 (BV && BV->isConstant());
4950 };
4951
4952 // All operands must be vector types with the same number of elements as
4953 // the result type and must be either UNDEF or a build vector of constant
4954 // or UNDEF scalars.
4955 if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
4956 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
4957 return SDValue();
4958
4959 // If we are comparing vectors, then the result needs to be a i1 boolean
4960 // that is then sign-extended back to the legal result type.
4961 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
4962
4963 // Find legal integer scalar type for constant promotion and
4964 // ensure that its scalar size is at least as large as source.
4965 EVT LegalSVT = VT.getScalarType();
4966 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
4967 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4968 if (LegalSVT.bitsLT(VT.getScalarType()))
4969 return SDValue();
4970 }
4971
4972 // Constant fold each scalar lane separately.
4973 SmallVector<SDValue, 4> ScalarResults;
4974 for (unsigned i = 0; i != NumElts; i++) {
4975 SmallVector<SDValue, 4> ScalarOps;
4976 for (SDValue Op : Ops) {
4977 EVT InSVT = Op.getValueType().getScalarType();
4978 BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
4979 if (!InBV) {
4980 // We've checked that this is UNDEF or a constant of some kind.
4981 if (Op.isUndef())
4982 ScalarOps.push_back(getUNDEF(InSVT));
4983 else
4984 ScalarOps.push_back(Op);
4985 continue;
4986 }
4987
4988 SDValue ScalarOp = InBV->getOperand(i);
4989 EVT ScalarVT = ScalarOp.getValueType();
4990
4991 // Build vector (integer) scalar operands may need implicit
4992 // truncation - do this before constant folding.
4993 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
4994 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
4995
4996 ScalarOps.push_back(ScalarOp);
4997 }
4998
4999 // Constant fold the scalar operands.
5000 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
5001
5002 // Legalize the (integer) scalar constant if necessary.
5003 if (LegalSVT != SVT)
5004 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
5005
5006 // Scalar folding only succeeded if the result is a constant or UNDEF.
5007 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
5008 ScalarResult.getOpcode() != ISD::ConstantFP)
5009 return SDValue();
5010 ScalarResults.push_back(ScalarResult);
5011 }
5012
5013 SDValue V = getBuildVector(VT, DL, ScalarResults);
5014 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
5015 return V;
5016}
5017
5018SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
5019 EVT VT, SDValue N1, SDValue N2) {
5020 // TODO: We don't do any constant folding for strict FP opcodes here, but we
5021 // should. That will require dealing with a potentially non-default
5022 // rounding mode, checking the "opStatus" return value from the APFloat
5023 // math calculations, and possibly other variations.
5024 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
5025 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
5026 if (N1CFP && N2CFP) {
5027 APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
5028 switch (Opcode) {
5029 case ISD::FADD:
5030 C1.add(C2, APFloat::rmNearestTiesToEven);
5031 return getConstantFP(C1, DL, VT);
5032 case ISD::FSUB:
5033 C1.subtract(C2, APFloat::rmNearestTiesToEven);
5034 return getConstantFP(C1, DL, VT);
5035 case ISD::FMUL:
5036 C1.multiply(C2, APFloat::rmNearestTiesToEven);
5037 return getConstantFP(C1, DL, VT);
5038 case ISD::FDIV:
5039 C1.divide(C2, APFloat::rmNearestTiesToEven);
5040 return getConstantFP(C1, DL, VT);
5041 case ISD::FREM:
5042 C1.mod(C2);
5043 return getConstantFP(C1, DL, VT);
5044 case ISD::FCOPYSIGN:
5045 C1.copySign(C2);
5046 return getConstantFP(C1, DL, VT);
5047 default: break;
5048 }
5049 }
5050 if (N1CFP && Opcode == ISD::FP_ROUND) {
5051 APFloat C1 = N1CFP->getValueAPF(); // make copy
5052 bool Unused;
5053 // This can return overflow, underflow, or inexact; we don't care.
5054 // FIXME need to be more flexible about rounding mode.
5055 (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5056 &Unused);
5057 return getConstantFP(C1, DL, VT);
5058 }
5059
5060 switch (Opcode) {
5061 case ISD::FADD:
5062 case ISD::FSUB:
5063 case ISD::FMUL:
5064 case ISD::FDIV:
5065 case ISD::FREM:
5066 // If both operands are undef, the result is undef. If 1 operand is undef,
5067 // the result is NaN. This should match the behavior of the IR optimizer.
5068 if (N1.isUndef() && N2.isUndef())
5069 return getUNDEF(VT);
5070 if (N1.isUndef() || N2.isUndef())
5071 return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5072 }
5073 return SDValue();
5074}
5075
5076SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5077 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5078 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5079 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5080 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5081 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5082
5083 // Canonicalize constant to RHS if commutative.
5084 if (TLI->isCommutativeBinOp(Opcode)) {
5085 if (N1C && !N2C) {
5086 std::swap(N1C, N2C);
5087 std::swap(N1, N2);
5088 } else if (N1CFP && !N2CFP) {
5089 std::swap(N1CFP, N2CFP);
5090 std::swap(N1, N2);
5091 }
5092 }
5093
5094 switch (Opcode) {
5095 default: break;
5096 case ISD::TokenFactor:
5097 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
5098 N2.getValueType() == MVT::Other && "Invalid token factor!");
5099 // Fold trivial token factors.
5100 if (N1.getOpcode() == ISD::EntryToken) return N2;
5101 if (N2.getOpcode() == ISD::EntryToken) return N1;
5102 if (N1 == N2) return N1;
5103 break;
5104 case ISD::BUILD_VECTOR: {
5105 // Attempt to simplify BUILD_VECTOR.
5106 SDValue Ops[] = {N1, N2};
5107 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5108 return V;
5109 break;
5110 }
5111 case ISD::CONCAT_VECTORS: {
5112 SDValue Ops[] = {N1, N2};
5113 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5114 return V;
5115 break;
5116 }
5117 case ISD::AND:
5118 assert(VT.isInteger() && "This operator does not apply to FP types!");
5119 assert(N1.getValueType() == N2.getValueType() &&
5120 N1.getValueType() == VT && "Binary operator types must match!");
5121 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
5122 // worth handling here.
5123 if (N2C && N2C->isNullValue())
5124 return N2;
5125 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
5126 return N1;
5127 break;
5128 case ISD::OR:
5129 case ISD::XOR:
5130 case ISD::ADD:
5131 case ISD::SUB:
5132 assert(!VT.isFatPointer() &&
5133 "This operator does not apply to capability types!");
5134 assert(VT.isInteger() && "This operator does not apply to FP types!");
5135 assert(N1.getValueType() == N2.getValueType() &&
5136 N1.getValueType() == VT && "Binary operator types must match!");
5137 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
5138 // it's worth handling here.
5139 if (N2C && N2C->isNullValue())
5140 return N1;
5141 break;
5142 case ISD::UDIV:
5143 case ISD::UREM:
5144 case ISD::MULHU:
5145 case ISD::MULHS:
5146 case ISD::MUL:
5147 case ISD::SDIV:
5148 case ISD::SREM:
5149 case ISD::SMIN:
5150 case ISD::SMAX:
5151 case ISD::UMIN:
5152 case ISD::UMAX:
5153 case ISD::SADDSAT:
5154 case ISD::SSUBSAT:
5155 case ISD::UADDSAT:
5156 case ISD::USUBSAT:
5157 assert(VT.isInteger() && "This operator does not apply to FP types!");
5158 assert(N1.getValueType() == N2.getValueType() &&
5159 N1.getValueType() == VT && "Binary operator types must match!");
5160 break;
5161 case ISD::FADD:
5162 case ISD::FSUB:
5163 case ISD::FMUL:
5164 case ISD::FDIV:
5165 case ISD::FREM:
5166 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
5167 assert(N1.getValueType() == N2.getValueType() &&
5168 N1.getValueType() == VT && "Binary operator types must match!");
5169 if (SDValue V = simplifyFPBinop(Opcode, N1, N2))
5170 return V;
5171 break;
5172 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
5173 assert(N1.getValueType() == VT &&
5174 N1.getValueType().isFloatingPoint() &&
5175 N2.getValueType().isFloatingPoint() &&
5176 "Invalid FCOPYSIGN!");
5177 break;
5178 case ISD::SHL:
5179 case ISD::SRA:
5180 case ISD::SRL:
5181 if (SDValue V = simplifyShift(N1, N2))
5182 return V;
5183 LLVM_FALLTHROUGH;
5184 case ISD::ROTL:
5185 case ISD::ROTR:
5186 assert(VT == N1.getValueType() &&
5187 "Shift operators return type must be the same as their first arg");
5188 assert(VT.isInteger() && N2.getValueType().isInteger() &&
5189 "Shifts only work on integers");
5190 assert((!VT.isVector() || VT == N2.getValueType()) &&
5191 "Vector shift amounts must be in the same as their first arg");
5192 // Verify that the shift amount VT is big enough to hold valid shift
5193 // amounts. This catches things like trying to shift an i1024 value by an
5194 // i8, which is easy to fall into in generic code that uses
5195 // TLI.getShiftAmount().
5196 assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
5197 "Invalid use of small shift amount with oversized value!");
5198
5199 // Always fold shifts of i1 values so the code generator doesn't need to
5200 // handle them. Since we know the size of the shift has to be less than the
5201 // size of the value, the shift/rotate count is guaranteed to be zero.
5202 if (VT == MVT::i1)
5203 return N1;
5204 if (N2C && N2C->isNullValue())
5205 return N1;
5206 break;
5207 case ISD::FP_ROUND_INREG: {
5208 EVT EVT = cast<VTSDNode>(N2)->getVT();
5209 assert(VT == N1.getValueType() && "Not an inreg round!");
5210 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
5211 "Cannot FP_ROUND_INREG integer types");
5212 assert(EVT.isVector() == VT.isVector() &&
5213 "FP_ROUND_INREG type should be vector iff the operand "
5214 "type is vector!");
5215 assert((!EVT.isVector() ||
5216 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
5217 "Vector element counts must match in FP_ROUND_INREG");
5218 assert(EVT.bitsLE(VT) && "Not rounding down!");
5219 (void)EVT;
5220 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
5221 break;
5222 }
5223 case ISD::FP_ROUND:
5224 assert(VT.isFloatingPoint() &&
5225 N1.getValueType().isFloatingPoint() &&
5226 VT.bitsLE(N1.getValueType()) &&
5227 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
5228 "Invalid FP_ROUND!");
5229 if (N1.getValueType() == VT) return N1; // noop conversion.
5230 break;
5231 case ISD::AssertSext:
5232 case ISD::AssertZext: {
5233 EVT EVT = cast<VTSDNode>(N2)->getVT();
5234 assert(VT == N1.getValueType() && "Not an inreg extend!");
5235 assert(VT.isInteger() && EVT.isInteger() &&
5236 "Cannot *_EXTEND_INREG FP types");
5237 assert(!EVT.isVector() &&
5238 "AssertSExt/AssertZExt type should be the vector element type "
5239 "rather than the vector type!");
5240 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
5241 if (VT.getScalarType() == EVT) return N1; // noop assertion.
5242 break;
5243 }
5244 case ISD::SIGN_EXTEND_INREG: {
5245 EVT EVT = cast<VTSDNode>(N2)->getVT();
5246 assert(VT == N1.getValueType() && "Not an inreg extend!");
5247 assert(VT.isInteger() && EVT.isInteger() &&
5248 "Cannot *_EXTEND_INREG FP types");
5249 assert(EVT.isVector() == VT.isVector() &&
5250 "SIGN_EXTEND_INREG type should be vector iff the operand "
5251 "type is vector!");
5252 assert((!EVT.isVector() ||
5253 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
5254 "Vector element counts must match in SIGN_EXTEND_INREG");
5255 assert(EVT.bitsLE(VT) && "Not extending!");
5256 if (EVT == VT) return N1; // Not actually extending
5257
5258 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
5259 unsigned FromBits = EVT.getScalarSizeInBits();
5260 Val <<= Val.getBitWidth() - FromBits;
5261 Val.ashrInPlace(Val.getBitWidth() - FromBits);
5262 return getConstant(Val, DL, ConstantVT);
5263 };
5264
5265 if (N1C) {
5266 const APInt &Val = N1C->getAPIntValue();
5267 return SignExtendInReg(Val, VT);
5268 }
5269 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
5270 SmallVector<SDValue, 8> Ops;
5271 llvm::EVT OpVT = N1.getOperand(0).getValueType();
5272 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
5273 SDValue Op = N1.getOperand(i);
5274 if (Op.isUndef()) {
5275 Ops.push_back(getUNDEF(OpVT));
5276 continue;
5277 }
5278 ConstantSDNode *C = cast<ConstantSDNode>(Op);
5279 APInt Val = C->getAPIntValue();
5280 Ops.push_back(SignExtendInReg(Val, OpVT));
5281 }
5282 return getBuildVector(VT, DL, Ops);
5283 }
5284 break;
5285 }
5286 case ISD::EXTRACT_VECTOR_ELT:
5287 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
5288 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
5289 element type of the vector.");
5290
5291 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
5292 if (N1.isUndef())
5293 return getUNDEF(VT);
5294
5295 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
5296 if (N2C && N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
5297 return getUNDEF(VT);
5298
5299 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
5300 // expanding copies of large vectors from registers.
5301 if (N2C &&
5302 N1.getOpcode() == ISD::CONCAT_VECTORS &&
5303 N1.getNumOperands() > 0) {
5304 unsigned Factor =
5305 N1.getOperand(0).getValueType().getVectorNumElements();
5306 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
5307 N1.getOperand(N2C->getZExtValue() / Factor),
5308 getConstant(N2C->getZExtValue() % Factor, DL,
5309 N2.getValueType()));
5310 }
5311
5312 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
5313 // expanding large vector constants.
5314 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
5315 SDValue Elt = N1.getOperand(N2C->getZExtValue());
5316
5317 if (VT != Elt.getValueType())
5318 // If the vector element type is not legal, the BUILD_VECTOR operands
5319 // are promoted and implicitly truncated, and the result implicitly
5320 // extended. Make that explicit here.
5321 Elt = getAnyExtOrTrunc(Elt, DL, VT);
5322
5323 return Elt;
5324 }
5325
5326 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
5327 // operations are lowered to scalars.
5328 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
5329 // If the indices are the same, return the inserted element else
5330 // if the indices are known different, extract the element from
5331 // the original vector.
5332 SDValue N1Op2 = N1.getOperand(2);
5333 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
5334
5335 if (N1Op2C && N2C) {
5336 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
5337 if (VT == N1.getOperand(1).getValueType())
5338 return N1.getOperand(1);
5339 else
5340 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
5341 }
5342
5343 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
5344 }
5345 }
5346
5347 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
5348 // when vector types are scalarized and v1iX is legal.
5349 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
5350 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5351 N1.getValueType().getVectorNumElements() == 1) {
5352 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
5353 N1.getOperand(1));
5354 }
5355 break;
5356 case ISD::EXTRACT_ELEMENT:
5357 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
5358 assert(!N1.getValueType().isVector() && !VT.isVector() &&
5359 (N1.getValueType().isInteger() == VT.isInteger()) &&
5360 N1.getValueType() != VT &&
5361 "Wrong types for EXTRACT_ELEMENT!");
5362
5363 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
5364 // 64-bit integers into 32-bit parts. Instead of building the extract of
5365 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
5366 if (N1.getOpcode() == ISD::BUILD_PAIR)
5367 return N1.getOperand(N2C->getZExtValue());
5368
5369 // EXTRACT_ELEMENT of a constant int is also very common.
5370 if (N1C) {
5371 unsigned ElementSize = VT.getSizeInBits();
5372 unsigned Shift = ElementSize * N2C->getZExtValue();
5373 APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
5374 return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
5375 }
5376 break;
5377 case ISD::EXTRACT_SUBVECTOR:
5378 if (VT.isSimple() && N1.getValueType().isSimple()) {
5379 assert(VT.isVector() && N1.getValueType().isVector() &&
5380 "Extract subvector VTs must be a vectors!");
5381 assert(VT.getVectorElementType() ==
5382 N1.getValueType().getVectorElementType() &&
5383 "Extract subvector VTs must have the same element type!");
5384 assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
5385 "Extract subvector must be from larger vector to smaller vector!");
5386
5387 if (N2C) {
5388 assert((VT.getVectorNumElements() + N2C->getZExtValue()
5389 <= N1.getValueType().getVectorNumElements())
5390 && "Extract subvector overflow!");
5391 }
5392
5393 // Trivial extraction.
5394 if (VT.getSimpleVT() == N1.getSimpleValueType())
5395 return N1;
5396
5397 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
5398 if (N1.isUndef())
5399 return getUNDEF(VT);
5400
5401 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
5402 // the concat have the same type as the extract.
5403 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
5404 N1.getNumOperands() > 0 &&
5405 VT == N1.getOperand(0).getValueType()) {
5406 unsigned Factor = VT.getVectorNumElements();
5407 return N1.getOperand(N2C->getZExtValue() / Factor);
5408 }
5409
5410 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
5411 // during shuffle legalization.
5412 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
5413 VT == N1.getOperand(1).getValueType())
5414 return N1.getOperand(1);
5415 }
5416 break;
5417 }
5418
5419 // Perform trivial constant folding.
5420 if (SDValue SV =
5421 FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
5422 return SV;
5423
5424 if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2))
5425 return V;
5426
5427 // Canonicalize an UNDEF to the RHS, even over a constant.
5428 if (N1.isUndef()) {
5429 if (TLI->isCommutativeBinOp(Opcode)) {
5430 std::swap(N1, N2);
5431 } else {
5432 switch (Opcode) {
5433 case ISD::FP_ROUND_INREG:
5434 case ISD::SIGN_EXTEND_INREG:
5435 case ISD::SUB:
5436 return getUNDEF(VT); // fold op(undef, arg2) -> undef
5437 case ISD::UDIV:
5438 case ISD::SDIV:
5439 case ISD::UREM:
5440 case ISD::SREM:
5441 case ISD::SSUBSAT:
5442 case ISD::USUBSAT:
5443 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
5444 }
5445 }
5446 }
5447
5448 // Fold a bunch of operators when the RHS is undef.
5449 if (N2.isUndef()) {
5450 switch (Opcode) {
5451 case ISD::XOR:
5452 if (N1.isUndef())
5453 // Handle undef ^ undef -> 0 special case. This is a common
5454 // idiom (misuse).
5455 return getConstant(0, DL, VT);
5456 LLVM_FALLTHROUGH;
5457 case ISD::ADD:
5458 case ISD::SUB:
5459 case ISD::UDIV:
5460 case ISD::SDIV:
5461 case ISD::UREM:
5462 case ISD::SREM:
5463 return getUNDEF(VT); // fold op(arg1, undef) -> undef
5464 case ISD::MUL:
5465 case ISD::AND:
5466 case ISD::SSUBSAT:
5467 case ISD::USUBSAT:
5468 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
5469 case ISD::OR:
5470 case ISD::SADDSAT:
5471 case ISD::UADDSAT:
5472 return getAllOnesConstant(DL, VT);
5473 }
5474 }
5475
5476 // Memoize this node if possible.
5477 SDNode *N;
5478 SDVTList VTs = getVTList(VT);
5479 SDValue Ops[] = {N1, N2};
5480 if (VT != MVT::Glue) {
5481 FoldingSetNodeID ID;
5482 AddNodeIDNode(ID, Opcode, VTs, Ops);
5483 void *IP = nullptr;
5484 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5485 E->intersectFlagsWith(Flags);
5486 return SDValue(E, 0);
5487 }
5488
5489 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5490 N->setFlags(Flags);
5491 createOperands(N, Ops);
5492 CSEMap.InsertNode(N, IP);
5493 } else {
5494 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5495 createOperands(N, Ops);
5496 }
5497
5498 InsertNode(N);
5499 SDValue V = SDValue(N, 0);
5500 NewSDValueDbgMsg(V, "Creating new node: ", this);
5501 return V;
5502}
5503
5504SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5505 SDValue N1, SDValue N2, SDValue N3,
5506 const SDNodeFlags Flags) {
5507 // Perform various simplifications.
5508 switch (Opcode) {
5509 case ISD::FMA: {
5510 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
5511 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
5512 N3.getValueType() == VT && "FMA types must match!");
5513 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5514 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5515 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
5516 if (N1CFP && N2CFP && N3CFP) {
5517 APFloat V1 = N1CFP->getValueAPF();
5518 const APFloat &V2 = N2CFP->getValueAPF();
5519 const APFloat &V3 = N3CFP->getValueAPF();
5520 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
5521 return getConstantFP(V1, DL, VT);
5522 }
5523 break;
5524 }
5525 case ISD::BUILD_VECTOR: {
5526 // Attempt to simplify BUILD_VECTOR.
5527 SDValue Ops[] = {N1, N2, N3};
5528 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5529 return V;
5530 break;
5531 }
5532 case ISD::CONCAT_VECTORS: {
5533 SDValue Ops[] = {N1, N2, N3};
5534 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
5535 return V;
5536 break;
5537 }
5538 case ISD::SETCC: {
5539 assert(VT.isInteger() && "SETCC result type must be an integer!");
5540 assert(N1.getValueType() == N2.getValueType() &&
5541 "SETCC operands must have the same type!");
5542 assert(VT.isVector() == N1.getValueType().isVector() &&
5543 "SETCC type should be vector iff the operand type is vector!");
5544 assert((!VT.isVector() ||
5545 VT.getVectorNumElements() == N1.getValueType().getVectorNumElements()) &&
5546 "SETCC vector element counts must match!");
5547 // Use FoldSetCC to simplify SETCC's.
5548 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
5549 return V;
5550 // Vector constant folding.
5551 SDValue Ops[] = {N1, N2, N3};
5552 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
5553 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
5554 return V;
5555 }
5556 break;
5557 }
5558 case ISD::SELECT:
5559 case ISD::VSELECT:
5560 if (SDValue V = simplifySelect(N1, N2, N3))
5561 return V;
5562 break;
5563 case ISD::VECTOR_SHUFFLE:
5564 llvm_unreachable("should use getVectorShuffle constructor!");
5565 case ISD::INSERT_VECTOR_ELT: {
5566 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
5567 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
5568 if (N3C && N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
5569 return getUNDEF(VT);
5570 break;
5571 }
5572 case ISD::INSERT_SUBVECTOR: {
5573 // Inserting undef into undef is still undef.
5574 if (N1.isUndef() && N2.isUndef())
5575 return getUNDEF(VT);
5576 SDValue Index = N3;
5577 if (VT.isSimple() && N1.getValueType().isSimple()
5578 && N2.getValueType().isSimple()) {
5579 assert(VT.isVector() && N1.getValueType().isVector() &&
5580 N2.getValueType().isVector() &&
5581 "Insert subvector VTs must be a vectors");
5582 assert(VT == N1.getValueType() &&
5583 "Dest and insert subvector source types must match!");
5584 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
5585 "Insert subvector must be from smaller vector to larger vector!");
5586 if (isa<ConstantSDNode>(Index)) {
5587 assert((N2.getValueType().getVectorNumElements() +
5588 cast<ConstantSDNode>(Index)->getZExtValue()
5589 <= VT.getVectorNumElements())
5590 && "Insert subvector overflow!");
5591 }
5592
5593 // Trivial insertion.
5594 if (VT.getSimpleVT() == N2.getSimpleValueType())
5595 return N2;
5596 }
5597 break;
5598 }
5599 case ISD::BITCAST:
5600 // Fold bit_convert nodes from a type to themselves.
5601 if (N1.getValueType() == VT)
5602 return N1;
5603 break;
5604 }
5605
5606 // Memoize node if it doesn't produce a flag.
5607 SDNode *N;
5608 SDVTList VTs = getVTList(VT);
5609 SDValue Ops[] = {N1, N2, N3};
5610 if (VT != MVT::Glue) {
5611 FoldingSetNodeID ID;
5612 AddNodeIDNode(ID, Opcode, VTs, Ops);
5613 void *IP = nullptr;
5614 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5615 E->intersectFlagsWith(Flags);
5616 return SDValue(E, 0);
5617 }
5618
5619 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5620 N->setFlags(Flags);
5621 createOperands(N, Ops);
5622 CSEMap.InsertNode(N, IP);
5623 } else {
5624 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5625 createOperands(N, Ops);
5626 }
5627
5628 InsertNode(N);
5629 SDValue V = SDValue(N, 0);
5630 NewSDValueDbgMsg(V, "Creating new node: ", this);
5631 return V;
5632}
5633
5634SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5635 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
5636 SDValue Ops[] = { N1, N2, N3, N4 };
5637 return getNode(Opcode, DL, VT, Ops);
5638}
5639
5640SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5641 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
5642 SDValue N5) {
5643 SDValue Ops[] = { N1, N2, N3, N4, N5 };
5644 return getNode(Opcode, DL, VT, Ops);
5645}
5646
5647/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
5648/// the incoming stack arguments to be loaded from the stack.
5649SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
5650 SmallVector<SDValue, 8> ArgChains;
5651
5652 // Include the original chain at the beginning of the list. When this is
5653 // used by target LowerCall hooks, this helps legalize find the
5654 // CALLSEQ_BEGIN node.
5655 ArgChains.push_back(Chain);
5656
5657 // Add a chain value for each stack argument.
5658 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
5659 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
5660 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
5661 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
5662 if (FI->getIndex() < 0)
5663 ArgChains.push_back(SDValue(L, 1));
5664
5665 // Build a tokenfactor for all the chains.
5666 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
5667}
5668
5669/// getMemsetValue - Vectorized representation of the memset value
5670/// operand.
5671static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
5672 const SDLoc &dl) {
5673 assert(!Value.isUndef());
5674
5675 unsigned NumBits = VT.getScalarSizeInBits();
5676 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
5677 assert(C->getAPIntValue().getBitWidth() == 8);
5678 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
5679 if (VT.isInteger() || VT.isFatPointer()) {
5680 bool IsOpaque = VT.getSizeInBits() > 64 ||
5681 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
5682 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
5683 }
5684 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
5685 VT);
5686 }
5687
5688 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
5689 EVT IntVT = VT.getScalarType();
5690 if (!IntVT.isInteger())
5691 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
5692
5693 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
5694 if (NumBits > 8) {
5695 // Use a multiplication with 0x010101... to extend the input to the
5696 // required length.
5697 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
5698 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
5699 DAG.getConstant(Magic, dl, IntVT));
5700 }
5701
5702 if (VT != Value.getValueType() && !VT.isInteger())
5703 Value = DAG.getBitcast(VT.getScalarType(), Value);
5704 if (VT != Value.getValueType())
5705 Value = DAG.getSplatBuildVector(VT, dl, Value);
5706
5707 return Value;
5708}
5709
5710/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
5711/// used when a memcpy is turned into a memset when the source is a constant
5712/// string ptr.
5713static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
5714 const TargetLowering &TLI,
5715 const ConstantDataArraySlice &Slice) {
5716 // Handle vector with all elements zero.
5717 if (Slice.Array == nullptr) {
5718 if (VT.isFatPointer())
5719 return DAG.getNullCapability(dl, VT);
5720 if (VT.isInteger())
5721 return DAG.getConstant(0, dl, VT);
5722 else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
5723 return DAG.getConstantFP(0.0, dl, VT);
5724 else if (VT.isVector()) {
5725 unsigned NumElts = VT.getVectorNumElements();
5726 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
5727 return DAG.getNode(ISD::BITCAST, dl, VT,
5728 DAG.getConstant(0, dl,
5729 EVT::getVectorVT(*DAG.getContext(),
5730 EltVT, NumElts)));
5731 } else
5732 llvm_unreachable("Expected type!");
5733 }
5734
5735 assert(!VT.isVector() && "Can't handle vector type here!");
5736 unsigned NumVTBits = VT.getSizeInBits();
5737 unsigned NumVTBytes = NumVTBits / 8;
5738 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
5739
5740 APInt Val(NumVTBits, 0);
5741 if (DAG.getDataLayout().isLittleEndian()) {
5742 for (unsigned i = 0; i != NumBytes; ++i)
5743 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
5744 } else {
5745 for (unsigned i = 0; i != NumBytes; ++i)
5746 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
5747 }
5748
5749 // If the "cost" of materializing the integer immediate is less than the cost
5750 // of a load, then it is cost effective to turn the load into the immediate.
5751 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
5752 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
5753 return DAG.getConstant(Val, dl, VT);
5754 return SDValue(nullptr, 0);
5755}
5756
5757/// getMemBasePlusOffset - Returns base and offset node for the
5758///
5759SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
5760 const SDLoc &DL,
5761 const SDNodeFlags Flags) {
5762 return getPointerAdd(DL, Base, Offset, Flags);
5763}
5764
5765/// Returns true if memcpy source is constant data.
5766static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
5767 uint64_t SrcDelta = 0;
5768 GlobalAddressSDNode *G = nullptr;
5769 if (Src.getOpcode() == ISD::GlobalAddress)
5770 G = cast<GlobalAddressSDNode>(Src);
5771 else if (Src.getOpcode() == ISD::ADD &&
5772 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
5773 Src.getOperand(1).getOpcode() == ISD::Constant) {
5774 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
5775 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
5776 }
5777 if (!G)
5778 return false;
5779
5780 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
5781 SrcDelta + G->getOffset());
5782}
5783
5784static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
5785 // On Darwin, -Os means optimize for size without hurting performance, so
5786 // only really optimize for size when -Oz (MinSize) is used.
5787 if (MF.getTarget().getTargetTriple().isOSDarwin())
5788 return MF.getFunction().hasMinSize();
5789 return MF.getFunction().hasOptSize();
5790}
5791
5792static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
5793 SmallVector<SDValue, 32> &OutChains, unsigned From,
5794 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
5795 SmallVector<SDValue, 16> &OutStoreChains) {
5796 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
5797 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
5798 SmallVector<SDValue, 16> GluedLoadChains;
5799 for (unsigned i = From; i < To; ++i) {
5800 OutChains.push_back(OutLoadChains[i]);
5801 GluedLoadChains.push_back(OutLoadChains[i]);
5802 }
5803
5804 // Chain for all loads.
5805 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5806 GluedLoadChains);
5807
5808 for (unsigned i = From; i < To; ++i) {
5809 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
5810 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
5811 ST->getBasePtr(), ST->getMemoryVT(),
5812 ST->getMemOperand());
5813 OutChains.push_back(NewStore);
5814 }
5815}
5816
5817static void
5818diagnoseInefficientCheriMemOp(SelectionDAG &DAG, const DiagnosticLocation &Loc,
5819 const Twine &MemOp, CodeGenOpt::Level OptLevel,
5820 StringRef Type, unsigned Align, uint64_t Size,
5821 uint64_t CapSize) {
5822 assert(Align < CapSize);
5823 assert(Size >= CapSize);
5824 if (OptLevel == CodeGenOpt::None)
5825 return; // Don't bother warning about inefficient code at -O0
5826 // Skip the memcpy/memmove diag if we have already diagnosed something else
5827 if (Type == "!!<CHERI-NODIAG>!!")
5828 return;
5829
5830 DiagnosticInfoCheriInefficient Warning(
5831 DAG.getMachineFunction().getFunction(), Loc,
5832 MemOp + " operation with capability argument " + Type +
5833 " and underaligned destination (aligned to " + Twine(Align) +
5834 " bytes) may be inefficient or result in CHERI tags bits being "
5835 "stripped");
5836 DAG.getContext()->diagnose(Warning);
5837}
5838
5839static SDValue getMemcpyLoadsAndStores(
5840 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
5841 uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline,
5842 bool MustPreserveCheriCapabilities, MachinePointerInfo DstPtrInfo,
5843 MachinePointerInfo SrcPtrInfo, StringRef CopyTy,
5844 CodeGenOpt::Level OptLevel) {
5845 // Turn a memcpy of undef to nop.
5846 if (Src.isUndef())
5847 return Chain;
5848
5849 // Expand memcpy to a series of load and store ops if the size operand falls
5850 // below a certain threshold.
5851 // TODO: In the AlwaysInline case, if the size is big then generate a loop
5852 // rather than maybe a humongous number of loads and stores.
5853 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5854 const DataLayout &DL = DAG.getDataLayout();
5855 LLVMContext &C = *DAG.getContext();
5856 std::vector<EVT> MemOps;
5857 bool DstAlignCanChange = false;
5858 MachineFunction &MF = DAG.getMachineFunction();
5859 MachineFrameInfo &MFI = MF.getFrameInfo();
5860 bool OptSize = shouldLowerMemFuncForSize(MF);
5861 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5862 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
5863 DstAlignCanChange = true;
5864 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
5865 if (Align > SrcAlign)
5866 SrcAlign = Align;
5867 ConstantDataArraySlice Slice;
5868 bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
5869 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
5870 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
5871 const bool FoundLowering = TLI.findOptimalMemOpLowering(MemOps, Limit, Size,
5872 (DstAlignCanChange ? 0 : Align),
5873 (isZeroConstant ? 0 : SrcAlign),
5874 false, false, CopyFromConstant, true, MustPreserveCheriCapabilities,
5875 DstPtrInfo.getAddrSpace(),
5876 SrcPtrInfo.getAddrSpace(),
5877 MF.getFunction().getAttributes());
5878
5879 // Don't warn about inefficient memcpy if we reached the inline memcpy limit
5880 // Also don't warn about copies of less than CapSize
5881 // TODO: the frontend probably shouldn't emit must-preserve-tags for such
5882 // small memcpys
5883 auto CapTy = TLI.cheriCapabilityType();
5884 const uint64_t CapSize = CapTy.isValid() ? CapTy.getStoreSize() : 0;
5885 bool ReachedLimit = (CapSize * Limit) < Size;
5886 if (MustPreserveCheriCapabilities && !ReachedLimit && Size >= CapSize &&
5887 (!FoundLowering || !MemOps[0].isFatPointer())) {
5888 LLVM_DEBUG(
5889 dbgs() << " memcpy must preserve tags but value is not statically "
5890 "known to be sufficiently aligned -> using memcpy() call\n");
5891 if (AlwaysInline) {
5892 report_fatal_error("MustPreserveCheriCapabilities and AlwaysInline set "
5893 "but operation cannot be lowered to loads+stores!");
5894 }
5895 diagnoseInefficientCheriMemOp(DAG, dl.getDebugLoc(), "memcpy", OptLevel,
5896 CopyTy.empty() ? "<unknown type>" : CopyTy,
5897 std::max(1u, std::min(Align, SrcAlign)),
5898 Size, CapSize);
5899 return SDValue();
5900 }
5901 if (!FoundLowering)
5902 return SDValue();
5903
5904 if (DstAlignCanChange) {
5905 Type *Ty = MemOps[0].getTypeForEVT(C);
5906 LLVM_DEBUG(dbgs() << " DstAlignCanChange -> using type "; Ty->dump());
5907 unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
5908 LLVM_DEBUG(dbgs() << "\t->NewAlign = " << NewAlign << ", stack alignment="
5909 << DL.getStackAlignment() << "\n");
5910 if (MemOps[0].isFatPointer()) {
5911 assert(!DL.exceedsNaturalStackAlignment(NewAlign) &&
5912 "Stack not capability-aligned?");
5913 }
5914
5915 // Don't promote to an alignment that would require dynamic stack
5916 // realignment.
5917 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
5918 if (!TRI->needsStackRealignment(MF))
5919 while (NewAlign > Align &&
5920 DL.exceedsNaturalStackAlignment(NewAlign))
5921 NewAlign /= 2;
5922 if (MemOps[0].isFatPointer()) {
5923 assert(NewAlign == (unsigned)DL.getABITypeAlignment(Ty) &&
5924 "Stack not capability-aligned?");
5925 }
5926 if (NewAlign > Align) {
5927 // Give the stack frame object a larger alignment if needed.
5928 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5929 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5930 Align = NewAlign;
5931 }
5932 }
5933
5934 MachineMemOperand::Flags MMOFlags =
5935 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
5936 SmallVector<SDValue, 16> OutLoadChains;
5937 SmallVector<SDValue, 16> OutStoreChains;
5938 SmallVector<SDValue, 32> OutChains;
5939 unsigned NumMemOps = MemOps.size();
5940 uint64_t SrcOff = 0, DstOff = 0;
5941 for (unsigned i = 0; i != NumMemOps; ++i) {
5942 EVT VT = MemOps[i];
5943 unsigned VTSize = VT.getSizeInBits() / 8;
5944 SDValue Value, Store;
5945
5946 if (VTSize > Size) {
5947 // Issuing an unaligned load / store pair that overlaps with the previous
5948 // pair. Adjust the offset accordingly.
5949 assert(i == NumMemOps-1 && i != 0);
5950 SrcOff -= VTSize - Size;
5951 DstOff -= VTSize - Size;
5952 }
5953
5954 if (CopyFromConstant &&
5955 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
5956 // It's unlikely a store of a vector immediate can be done in a single
5957 // instruction. It would require a load from a constantpool first.
5958 // We only handle zero vectors here.
5959 // FIXME: Handle other cases where store of vector immediate is done in
5960 // a single instruction.
5961 ConstantDataArraySlice SubSlice;
5962 if (SrcOff < Slice.Length) {
5963 SubSlice = Slice;
5964 SubSlice.move(SrcOff);
5965 } else {
5966 // This is an out-of-bounds access and hence UB. Pretend we read zero.
5967 SubSlice.Array = nullptr;
5968 SubSlice.Offset = 0;
5969 SubSlice.Length = VTSize;
5970 }
5971 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
5972 if (Value.getNode()) {
5973 Store = DAG.getStore(Chain, dl, Value,
5974 DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5975 DstPtrInfo.getWithOffset(DstOff), Align,
5976 MMOFlags);
5977 OutChains.push_back(Store);
5978 }
5979 }
5980
5981 if (!Store.getNode()) {
5982 // The type might not be legal for the target. This should only happen
5983 // if the type is smaller than a legal type, as on PPC, so the right
5984 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
5985 // to Load/Store if NVT==VT.
5986 // FIXME does the case above also need this?
5987 EVT NVT = TLI.getTypeToTransformTo(C, VT);
5988 assert(NVT.bitsGE(VT));
5989
5990 bool isDereferenceable =
5991 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
5992 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
5993 if (isDereferenceable)
5994 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
5995
5996 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
5997 DAG.getMemBasePlusOffset(Src, SrcOff, dl),
5998 SrcPtrInfo.getWithOffset(SrcOff), VT,
5999 MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
6000 OutLoadChains.push_back(Value.getValue(1));
6001
6002 Store = DAG.getTruncStore(
6003 Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
6004 DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
6005 OutStoreChains.push_back(Store);
6006 }
6007 SrcOff += VTSize;
6008 DstOff += VTSize;
6009 Size -= VTSize;
6010 }
6011
6012 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
6013 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
6014 unsigned NumLdStInMemcpy = OutStoreChains.size();
6015
6016 if (NumLdStInMemcpy) {
6017 // It may be that memcpy might be converted to memset if it's memcpy
6018 // of constants. In such a case, we won't have loads and stores, but
6019 // just stores. In the absence of loads, there is nothing to gang up.
6020 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
6021 // If target does not care, just leave as it.
6022 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
6023 OutChains.push_back(OutLoadChains[i]);
6024 OutChains.push_back(OutStoreChains[i]);
6025 }
6026 } else {
6027 // Ld/St less than/equal limit set by target.
6028 if (NumLdStInMemcpy <= GluedLdStLimit) {
6029 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6030 NumLdStInMemcpy, OutLoadChains,
6031 OutStoreChains);
6032 } else {
6033 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
6034 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
6035 unsigned GlueIter = 0;
6036
6037 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
6038 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
6039 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
6040
6041 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
6042 OutLoadChains, OutStoreChains);
6043 GlueIter += GluedLdStLimit;
6044 }
6045
6046 // Residual ld/st.
6047 if (RemainingLdStInMemcpy) {
6048 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
6049 RemainingLdStInMemcpy, OutLoadChains,
6050 OutStoreChains);
6051 }
6052 }
6053 }
6054 }
6055 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6056}
6057
6058static SDValue getMemmoveLoadsAndStores(
6059 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
6060 uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline,
6061 bool MustPreserveCheriCapabilities, MachinePointerInfo DstPtrInfo,
6062 MachinePointerInfo SrcPtrInfo, StringRef MoveTy,
6063 CodeGenOpt::Level OptLevel) {
6064 // Turn a memmove of undef to nop.
6065 if (Src.isUndef())
6066 return Chain;
6067
6068 // Expand memmove to a series of load and store ops if the size operand falls
6069 // below a certain threshold.
6070 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6071 const DataLayout &DL = DAG.getDataLayout();
6072 LLVMContext &C = *DAG.getContext();
6073 std::vector<EVT> MemOps;
6074 bool DstAlignCanChange = false;
6075 MachineFunction &MF = DAG.getMachineFunction();
6076 MachineFrameInfo &MFI = MF.getFrameInfo();
6077 bool OptSize = shouldLowerMemFuncForSize(MF);
6078 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6079 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6080 DstAlignCanChange = true;
6081 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6082 if (Align > SrcAlign)
6083 SrcAlign = Align;
6084 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
6085 const bool FoundLowering = TLI.findOptimalMemOpLowering(MemOps, Limit, Size,
6086 (DstAlignCanChange ? 0 : Align), SrcAlign,
6087 false, false, false, false, MustPreserveCheriCapabilities,
6088 DstPtrInfo.getAddrSpace(),
6089 SrcPtrInfo.getAddrSpace(),
6090 MF.getFunction().getAttributes());
6091
6092 // Don't warn about inefficient memcpy if we reached the inline memmove limit
6093 // Also don't warn about copies of less than CapSize
6094 // TODO: the frontend probably shouldn't emit must-preserve-tags for such
6095 // small memcpys
6096 auto CapTy = TLI.cheriCapabilityType();
6097 const uint64_t CapSize = CapTy.isValid() ? CapTy.getStoreSize() : 0;
6098 bool ReachedLimit = (CapSize * Limit) < Size;
6099 if (MustPreserveCheriCapabilities && !ReachedLimit && Size >= CapSize &&
6100 (!FoundLowering || !MemOps[0].isFatPointer())) {
6101 LLVM_DEBUG(
6102 dbgs() << __func__
6103 << " memmove must preserve tags but value is not statically "
6104 "known to be sufficiently aligned -> using memmove() call\n");
6105 if (AlwaysInline) {
6106 report_fatal_error("MustPreserveCheriCapabilities and AlwaysInline set "
6107 "but operation cannot be lowered to loads+stores!");
6108 }
6109 diagnoseInefficientCheriMemOp(DAG, dl.getDebugLoc(), "memmove", OptLevel,
6110 MoveTy.empty() ? "<unknown type>" : MoveTy,
6111 std::max(1u, std::min(Align, SrcAlign)),
6112 Size, CapSize);
6113 return SDValue();
6114 }
6115 if (!FoundLowering)
6116 return SDValue();
6117
6118 if (DstAlignCanChange) {
6119 Type *Ty = MemOps[0].getTypeForEVT(C);
6120 unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
6121 if (MemOps[0].isFatPointer()) {
6122 assert(!DL.exceedsNaturalStackAlignment(NewAlign) &&
6123 "Stack not capability-aligned?");
6124 }
6125 if (NewAlign > Align) {
6126 // Give the stack frame object a larger alignment if needed.
6127 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
6128 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6129 Align = NewAlign;
6130 }
6131 }
6132
6133 MachineMemOperand::Flags MMOFlags =
6134 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
6135 uint64_t SrcOff = 0, DstOff = 0;
6136 SmallVector<SDValue, 8> LoadValues;
6137 SmallVector<SDValue, 8> LoadChains;
6138 SmallVector<SDValue, 8> OutChains;
6139 unsigned NumMemOps = MemOps.size();
6140 for (unsigned i = 0; i < NumMemOps; i++) {
6141 EVT VT = MemOps[i];
6142 unsigned VTSize = VT.getSizeInBits() / 8;
6143 SDValue Value;
6144
6145 bool isDereferenceable =
6146 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
6147 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
6148 if (isDereferenceable)
6149 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
6150
6151 Value =
6152 DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl),
6153 SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags);
6154 LoadValues.push_back(Value);
6155 LoadChains.push_back(Value.getValue(1));
6156 SrcOff += VTSize;
6157 }
6158 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
6159 OutChains.clear();
6160 for (unsigned i = 0; i < NumMemOps; i++) {
6161 EVT VT = MemOps[i];
6162 unsigned VTSize = VT.getSizeInBits() / 8;
6163 SDValue Store;
6164
6165 Store = DAG.getStore(Chain, dl, LoadValues[i],
6166 DAG.getMemBasePlusOffset(Dst, DstOff, dl),
6167 DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags);
6168 OutChains.push_back(Store);
6169 DstOff += VTSize;
6170 }
6171
6172 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6173}
6174
6175/// Lower the call to 'memset' intrinsic function into a series of store
6176/// operations.
6177///
6178/// \param DAG Selection DAG where lowered code is placed.
6179/// \param dl Link to corresponding IR location.
6180/// \param Chain Control flow dependency.
6181/// \param Dst Pointer to destination memory location.
6182/// \param Src Value of byte to write into the memory.
6183/// \param Size Number of bytes to write.
6184/// \param Align Alignment of the destination in bytes.
6185/// \param isVol True if destination is volatile.
6186/// \param DstPtrInfo IR information on the memory pointer.
6187/// \returns New head in the control flow, if lowering was successful, empty
6188/// SDValue otherwise.
6189///
6190/// The function tries to replace 'llvm.memset' intrinsic with several store
6191/// operations and value calculation code. This is usually profitable for small
6192/// memory size.
6193static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
6194 SDValue Chain, SDValue Dst, SDValue Src,
6195 uint64_t Size, unsigned Align, bool isVol,
6196 MachinePointerInfo DstPtrInfo) {
6197 // Turn a memset of undef to nop.
6198 if (Src.isUndef())
6199 return Chain;
6200
6201 // Expand memset to a series of load/store ops if the size operand
6202 // falls below a certain threshold.
6203 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6204 std::vector<EVT> MemOps;
6205 bool DstAlignCanChange = false;
6206 MachineFunction &MF = DAG.getMachineFunction();
6207 MachineFrameInfo &MFI = MF.getFrameInfo();
6208 bool OptSize = shouldLowerMemFuncForSize(MF);
6209 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
6210 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
6211 DstAlignCanChange = true;
6212 bool IsZeroVal =
6213 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
6214 if (!TLI.findOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
6215 Size, (DstAlignCanChange ? 0 : Align), 0,
6216 true, IsZeroVal, false, true, false,
6217 DstPtrInfo.getAddrSpace(), ~0u,
6218 MF.getFunction().getAttributes()))
6219 return SDValue();
6220
6221 if (DstAlignCanChange) {
6222 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
6223 unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
6224 if (MemOps[0].isFatPointer()) {
6225 assert(!DAG.getDataLayout().exceedsNaturalStackAlignment(NewAlign) &&
6226 "Stack not capability-aligned?");
6227 }
6228 if (NewAlign > Align) {
6229 // Give the stack frame object a larger alignment if needed.
6230 if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
6231 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
6232 Align = NewAlign;
6233 }
6234 }
6235
6236 SmallVector<SDValue, 8> OutChains;
6237 uint64_t DstOff = 0;
6238 unsigned NumMemOps = MemOps.size();
6239
6240 // Find the largest store and generate the bit pattern for it.
6241 EVT LargestVT = MemOps[0];
6242 for (unsigned i = 1; i < NumMemOps; i++)
6243 if (MemOps[i].bitsGT(LargestVT))
6244 LargestVT = MemOps[i];
6245 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
6246
6247 for (unsigned i = 0; i < NumMemOps; i++) {
6248 EVT VT = MemOps[i];
6249 unsigned VTSize = VT.getSizeInBits() / 8;
6250 if (VTSize > Size) {
6251 // Issuing an unaligned load / store pair that overlaps with the previous
6252 // pair. Adjust the offset accordingly.
6253 assert(i == NumMemOps-1 && i != 0);
6254 DstOff -= VTSize - Size;
6255 }
6256
6257 // If this store is smaller than the largest store see whether we can get
6258 // the smaller value for free with a truncate.
6259 SDValue Value = MemSetValue;
6260 if (VT.bitsLT(LargestVT)) {
6261 if (!LargestVT.isVector() && !VT.isVector() &&
6262 TLI.isTruncateFree(LargestVT, VT))
6263 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
6264 else
6265 Value = getMemsetValue(Src, VT, DAG, dl);
6266 }
6267 assert(Value.getValueType() == VT && "Value with wrong type.");
6268 SDValue Store = DAG.getStore(
6269 Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
6270 DstPtrInfo.getWithOffset(DstOff), Align,
6271 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone);
6272 OutChains.push_back(Store);
6273 DstOff += VT.getSizeInBits() / 8;
6274 Size -= VTSize;
6275 }
6276
6277 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
6278}
6279
6280SDValue SelectionDAG::getPointerAdd(const SDLoc dl, SDValue Ptr, SDValue Offset,
6281 const SDNodeFlags Flags) {
6282 assert(Offset.getValueType().isInteger());
6283 EVT BasePtrVT = Ptr.getValueType();
6284 if (BasePtrVT.isFatPointer()) {
6285 if (auto *Constant = dyn_cast<ConstantSDNode>(Offset.getNode())) {
6286 if (Constant->isNullValue())
6287 return Ptr;
6288 }
6289 return getNode(ISD::PTRADD, dl, BasePtrVT, Ptr, Offset, Flags);
6290 }
6291 return getNode(ISD::ADD, dl, BasePtrVT, Ptr, Offset, Flags);
6292}
6293
6294SDValue SelectionDAG::getCSetBounds(SDValue Val, SDValue Length,
6295 bool CSetBoundsStatsAlreadyLogged) {
6296 if (!CSetBoundsStatsAlreadyLogged && cheri::ShouldCollectCSetBoundsStats) {
6297 Optional<uint64_t> SizeConst;
6298 if (ConstantSDNode *Constant = dyn_cast<ConstantSDNode>(Length.getNode())) {
6299 SizeConst = Constant->getZExtValue();
6300 }
6301 cheri::CSetBoundsStats->add(
6302 1, SizeConst, "SelectionDAG::getCSetBounds",
6303 cheri::SetBoundsPointerSource::Unknown, "<unnkonwn reason>",
6304 cheri::inferSourceLocation(Val.getDebugLoc(),
6305 getMachineFunction().getName()));
6306 }
6307 SDLoc DL(Val);
6308 Intrinsic::ID SetBounds = Intrinsic::cheri_cap_bounds_set;
6309 // Using the bounded stack cap intrinisic allows reuse of the same register:
6310 if (isa<FrameIndexSDNode>(Val.getNode()))
6311 SetBounds = Intrinsic::cheri_bounded_stack_cap;
6312 MVT SizeVT = MVT::getIntegerVT(getDataLayout().getPointerSizeInBits(0));
6313 return getNode(ISD::INTRINSIC_WO_CHAIN, DL, Val.getValueType(),
6314 getConstant(SetBounds, DL, SizeVT), Val, Length);
6315}
6316
6317static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
6318 unsigned AS) {
6319 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
6320 // pointer operands can be losslessly bitcasted to pointers of address space 0
6321 if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
6322 report_fatal_error("cannot lower memory intrinsic in address space " +
6323 Twine(AS));
6324 }
6325}
6326
6327SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
6328 SDValue Src, SDValue Size, unsigned Align,
6329 bool isVol, bool AlwaysInline, bool isTailCall,
6330 bool MustPreserveCheriCapabilities,
6331 MachinePointerInfo DstPtrInfo,
6332 MachinePointerInfo SrcPtrInfo,
6333 StringRef CopyType) {
6334 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
6335 LLVM_DEBUG(dbgs() << "DAG.getMemcpy() align=" << Align << " size=";
6336 Size.dump(););
6337
6338 // Check to see if we should lower the memcpy to loads and stores first.
6339 // For cases within the target-specified limits, this is the best choice.
6340 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6341 if (MustPreserveCheriCapabilities)
6342 assert(TLI->cheriCapabilityType().isValid());
6343 if (ConstantSize) {
6344 // Memcpy with size zero? Just return the original chain.
6345 if (ConstantSize->isNullValue())
6346 return Chain;
6347
6348 SDValue Result;
6349 Result = getMemcpyLoadsAndStores(
6350 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Align, isVol,
6351 false, MustPreserveCheriCapabilities, DstPtrInfo, SrcPtrInfo, CopyType,
6352 OptLevel);
6353 if (Result.getNode())
6354 return Result;
6355 }
6356
6357 // Then check to see if we should lower the memcpy with target-specific
6358 // code. If the target chooses to do this, this is the next best.
6359 if (TSI) {
6360 SDValue Result = TSI->EmitTargetCodeForMemcpy(
6361 *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
6362 MustPreserveCheriCapabilities, DstPtrInfo, SrcPtrInfo);
6363 if (Result.getNode())
6364 return Result;
6365 }
6366
6367 // If we really need inline code and the target declined to provide it,
6368 // use a (potentially long) sequence of loads and stores.
6369 if (AlwaysInline) {
6370 assert(ConstantSize && "AlwaysInline requires a constant size!");
6371 return getMemcpyLoadsAndStores(
6372 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Align, isVol,
6373 true, MustPreserveCheriCapabilities, DstPtrInfo, SrcPtrInfo, CopyType,
6374 OptLevel);
6375 }
6376
6377 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6378 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6379
6380 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
6381 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
6382 // respect volatile, so they may do things like read or write memory
6383 // beyond the given memory regions. But fixing this isn't easy, and most
6384 // people don't care.
6385
6386 // Emit a library call.
6387 TargetLowering::ArgListTy Args;
6388 TargetLowering::ArgListEntry Entry;
6389 Entry.Ty = Dst.getValueType().getTypeForEVT(*getContext());
6390 Entry.Node = Dst; Args.push_back(Entry);
6391 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
6392 Entry.Node = Src; Args.push_back(Entry);
6393
6394 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6395 Entry.Node = Size; Args.push_back(Entry);
6396 // FIXME: pass in SDLoc
6397 TargetLowering::CallLoweringInfo CLI(*this);
6398 CLI.setDebugLoc(dl)
6399 .setChain(Chain)
6400 .setLibCallee(
6401 TLI->getLibcallCallingConv(RTLIB::MEMCPY),
6402 Dst.getValueType().getTypeForEVT(*getContext()),
6403 getExternalFunctionSymbol(TLI->getLibcallName(RTLIB::MEMCPY)),
6404 std::move(Args))
6405 .setDiscardResult()
6406 .setTailCall(isTailCall);
6407
6408 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6409 return CallResult.second;
6410}
6411
6412SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
6413 SDValue Dst, unsigned DstAlign,
6414 SDValue Src, unsigned SrcAlign,
6415 SDValue Size, Type *SizeTy,
6416 unsigned ElemSz, bool isTailCall,
6417 MachinePointerInfo DstPtrInfo,
6418 MachinePointerInfo SrcPtrInfo) {
6419 // Emit a library call.
6420 TargetLowering::ArgListTy Args;
6421 TargetLowering::ArgListEntry Entry;
6422 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6423 Entry.Node = Dst;
6424 Args.push_back(Entry);
6425
6426 Entry.Node = Src;
6427 Args.push_back(Entry);
6428
6429 Entry.Ty = SizeTy;
6430 Entry.Node = Size;
6431 Args.push_back(Entry);
6432
6433 RTLIB::Libcall LibraryCall =
6434 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6435 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6436 report_fatal_error("Unsupported element size");
6437
6438 TargetLowering::CallLoweringInfo CLI(*this);
6439 CLI.setDebugLoc(dl)
6440 .setChain(Chain)
6441 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6442 Type::getVoidTy(*getContext()),
6443 getExternalFunctionSymbol(TLI->getLibcallName(LibraryCall)),
6444 std::move(Args))
6445 .setDiscardResult()
6446 .setTailCall(isTailCall);
6447
6448 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6449 return CallResult.second;
6450}
6451
6452SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
6453 SDValue Src, SDValue Size, unsigned Align,
6454 bool isVol, bool isTailCall,
6455 bool MustPreserveCheriCapabilities,
6456 MachinePointerInfo DstPtrInfo,
6457 MachinePointerInfo SrcPtrInfo,
6458 StringRef MoveType) {
6459 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
6460
6461 // Check to see if we should lower the memmove to loads and stores first.
6462 // For cases within the target-specified limits, this is the best choice.
6463 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6464 if (ConstantSize) {
6465 // Memmove with size zero? Just return the original chain.
6466 if (ConstantSize->isNullValue())
6467 return Chain;
6468
6469 SDValue Result;
6470 Result = getMemmoveLoadsAndStores(
6471 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Align, isVol,
6472 false, MustPreserveCheriCapabilities, DstPtrInfo, SrcPtrInfo, MoveType,
6473 OptLevel);
6474 if (Result.getNode())
6475 return Result;
6476 }
6477
6478 // Then check to see if we should lower the memmove with target-specific
6479 // code. If the target chooses to do this, this is the next best.
6480 if (TSI) {
6481 SDValue Result = TSI->EmitTargetCodeForMemmove(
6482 *this, dl, Chain, Dst, Src, Size, Align, isVol,
6483 MustPreserveCheriCapabilities, DstPtrInfo, SrcPtrInfo);
6484 if (Result.getNode())
6485 return Result;
6486 }
6487
6488 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6489 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
6490
6491 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
6492 // not be safe. See memcpy above for more details.
6493
6494 // Emit a library call.
6495 TargetLowering::ArgListTy Args;
6496 TargetLowering::ArgListEntry Entry;
6497 Entry.Ty = Dst.getValueType().getTypeForEVT(*getContext());
6498 Entry.Node = Dst; Args.push_back(Entry);
6499 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
6500 Entry.Node = Src; Args.push_back(Entry);
6501
6502 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6503 Entry.Node = Size; Args.push_back(Entry);
6504 // FIXME: pass in SDLoc
6505 TargetLowering::CallLoweringInfo CLI(*this);
6506 CLI.setDebugLoc(dl)
6507 .setChain(Chain)
6508 .setLibCallee(
6509 TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
6510 Dst.getValueType().getTypeForEVT(*getContext()),
6511 getExternalFunctionSymbol(TLI->getLibcallName(RTLIB::MEMMOVE)),
6512 std::move(Args))
6513 .setDiscardResult()
6514 .setTailCall(isTailCall);
6515
6516 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6517 return CallResult.second;
6518}
6519
6520SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
6521 SDValue Dst, unsigned DstAlign,
6522 SDValue Src, unsigned SrcAlign,
6523 SDValue Size, Type *SizeTy,
6524 unsigned ElemSz, bool isTailCall,
6525 MachinePointerInfo DstPtrInfo,
6526 MachinePointerInfo SrcPtrInfo) {
6527 // Emit a library call.
6528 TargetLowering::ArgListTy Args;
6529 TargetLowering::ArgListEntry Entry;
6530 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6531 Entry.Node = Dst;
6532 Args.push_back(Entry);
6533
6534 Entry.Node = Src;
6535 Args.push_back(Entry);
6536
6537 Entry.Ty = SizeTy;
6538 Entry.Node = Size;
6539 Args.push_back(Entry);
6540
6541 RTLIB::Libcall LibraryCall =
6542 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6543 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6544 report_fatal_error("Unsupported element size");
6545
6546 TargetLowering::CallLoweringInfo CLI(*this);
6547 CLI.setDebugLoc(dl)
6548 .setChain(Chain)
6549 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6550 Type::getVoidTy(*getContext()),
6551 getExternalFunctionSymbol(TLI->getLibcallName(LibraryCall)),
6552 std::move(Args))
6553 .setDiscardResult()
6554 .setTailCall(isTailCall);
6555
6556 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6557 return CallResult.second;
6558}
6559
6560SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
6561 SDValue Src, SDValue Size, unsigned Align,
6562 bool isVol, bool isTailCall,
6563 MachinePointerInfo DstPtrInfo) {
6564 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
6565
6566 // Check to see if we should lower the memset to stores first.
6567 // For cases within the target-specified limits, this is the best choice.
6568 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
6569 if (ConstantSize) {
6570 // Memset with size zero? Just return the original chain.
6571 if (ConstantSize->isNullValue())
6572 return Chain;
6573
6574 SDValue Result =
6575 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
6576 Align, isVol, DstPtrInfo);
6577
6578 if (Result.getNode())
6579 return Result;
6580 }
6581
6582 // Then check to see if we should lower the memset with target-specific
6583 // code. If the target chooses to do this, this is the next best.
6584 if (TSI) {
6585 SDValue Result = TSI->EmitTargetCodeForMemset(
6586 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
6587 if (Result.getNode())
6588 return Result;
6589 }
6590
6591 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
6592
6593 // Emit a library call.
6594 TargetLowering::ArgListTy Args;
6595 TargetLowering::ArgListEntry Entry;
6596 Entry.Node = Dst; Entry.Ty = Dst.getValueType().getTypeForEVT(*getContext());
6597 Args.push_back(Entry);
6598 Entry.Node = Src;
6599 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
6600 Args.push_back(Entry);
6601 Entry.Node = Size;
6602 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6603 Args.push_back(Entry);
6604
6605 // FIXME: pass in SDLoc
6606 TargetLowering::CallLoweringInfo CLI(*this);
6607 CLI.setDebugLoc(dl)
6608 .setChain(Chain)
6609 .setLibCallee(
6610 TLI->getLibcallCallingConv(RTLIB::MEMSET),
6611 Dst.getValueType().getTypeForEVT(*getContext()),
6612 getExternalFunctionSymbol(TLI->getLibcallName(RTLIB::MEMSET)),
6613 std::move(Args))
6614 .setDiscardResult()
6615 .setTailCall(isTailCall);
6616
6617 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
6618 return CallResult.second;
6619}
6620
6621SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
6622 SDValue Dst, unsigned DstAlign,
6623 SDValue Value, SDValue Size, Type *SizeTy,
6624 unsigned ElemSz, bool isTailCall,
6625 MachinePointerInfo DstPtrInfo) {
6626 // Emit a library call.
6627 TargetLowering::ArgListTy Args;
6628 TargetLowering::ArgListEntry Entry;
6629 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
6630 Entry.Node = Dst;
6631 Args.push_back(Entry);
6632
6633 Entry.Ty = Type::getInt8Ty(*getContext());
6634 Entry.Node = Value;
6635 Args.push_back(Entry);
6636
6637 Entry.Ty = SizeTy;
6638 Entry.Node = Size;
6639 Args.push_back(Entry);
6640
6641 RTLIB::Libcall LibraryCall =
6642 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
6643 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
6644 report_fatal_error("Unsupported element size");
6645
6646 TargetLowering::CallLoweringInfo CLI(*this);
6647 CLI.setDebugLoc(dl)
6648 .setChain(Chain)
6649 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
6650 Type::getVoidTy(*getContext()),
6651 getExternalFunctionSymbol(TLI->getLibcallName(LibraryCall)),
6652 std::move(Args))
6653 .setDiscardResult()
6654 .setTailCall(isTailCall);
6655
6656 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
6657 return CallResult.second;
6658}
6659
6660SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6661 SDVTList VTList, ArrayRef<SDValue> Ops,
6662 MachineMemOperand *MMO) {
6663 FoldingSetNodeID ID;
6664 ID.AddInteger(MemVT.getRawBits());
6665 AddNodeIDNode(ID, Opcode, VTList, Ops);
6666 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6667 void* IP = nullptr;
6668 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6669 cast<AtomicSDNode>(E)->refineAlignment(MMO);
6670 return SDValue(E, 0);
6671 }
6672
6673 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6674 VTList, MemVT, MMO);
6675 createOperands(N, Ops);
6676
6677 CSEMap.InsertNode(N, IP);
6678 InsertNode(N);
6679 return SDValue(N, 0);
6680}
6681
6682SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
6683 EVT MemVT, SDVTList VTs, SDValue Chain,
6684 SDValue Ptr, SDValue Cmp, SDValue Swp,
6685 MachineMemOperand *MMO) {
6686 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
6687 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
6688 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
6689
6690 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
6691 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6692}
6693
6694SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6695 SDValue Chain, SDValue Ptr, SDValue Val,
6696 MachineMemOperand *MMO) {
6697 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
6698 Opcode == ISD::ATOMIC_LOAD_SUB ||
6699 Opcode == ISD::ATOMIC_LOAD_AND ||
6700 Opcode == ISD::ATOMIC_LOAD_CLR ||
6701 Opcode == ISD::ATOMIC_LOAD_OR ||
6702 Opcode == ISD::ATOMIC_LOAD_XOR ||
6703 Opcode == ISD::ATOMIC_LOAD_NAND ||
6704 Opcode == ISD::ATOMIC_LOAD_MIN ||
6705 Opcode == ISD::ATOMIC_LOAD_MAX ||
6706 Opcode == ISD::ATOMIC_LOAD_UMIN ||
6707 Opcode == ISD::ATOMIC_LOAD_UMAX ||
6708 Opcode == ISD::ATOMIC_LOAD_FADD ||
6709 Opcode == ISD::ATOMIC_LOAD_FSUB ||
6710 Opcode == ISD::ATOMIC_SWAP ||
6711 Opcode == ISD::ATOMIC_STORE) &&
6712 "Invalid Atomic Op");
6713
6714 EVT VT = Val.getValueType();
6715
6716 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
6717 getVTList(VT, MVT::Other);
6718 SDValue Ops[] = {Chain, Ptr, Val};
6719 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6720}
6721
6722SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
6723 EVT VT, SDValue Chain, SDValue Ptr,
6724 MachineMemOperand *MMO) {
6725 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
6726
6727 SDVTList VTs = getVTList(VT, MVT::Other);
6728 SDValue Ops[] = {Chain, Ptr};
6729 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
6730}
6731
6732/// getMergeValues - Create a MERGE_VALUES node from the given operands.
6733SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
6734 if (Ops.size() == 1)
6735 return Ops[0];
6736
6737 SmallVector<EVT, 4> VTs;
6738 VTs.reserve(Ops.size());
6739 for (unsigned i = 0; i < Ops.size(); ++i)
6740 VTs.push_back(Ops[i].getValueType());
6741 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
6742}
6743
6744SDValue SelectionDAG::getMemIntrinsicNode(
6745 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
6746 EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align,
6747 MachineMemOperand::Flags Flags, unsigned Size) {
6748 if (Align == 0) // Ensure that codegen never sees alignment 0
6749 Align = getEVTAlignment(MemVT);
6750
6751 if (!Size)
6752 Size = MemVT.getStoreSize();
6753
6754 MachineFunction &MF = getMachineFunction();
6755 MachineMemOperand *MMO =
6756 MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
6757
6758 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
6759}
6760
6761SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
6762 SDVTList VTList,
6763 ArrayRef<SDValue> Ops, EVT MemVT,
6764 MachineMemOperand *MMO) {
6765 assert((Opcode == ISD::INTRINSIC_VOID ||
6766 Opcode == ISD::INTRINSIC_W_CHAIN ||
6767 Opcode == ISD::PREFETCH ||
6768 Opcode == ISD::LIFETIME_START ||
6769 Opcode == ISD::LIFETIME_END ||
6770 ((int)Opcode <= std::numeric_limits<int>::max() &&
6771 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
6772 "Opcode is not a memory-accessing opcode!");
6773
6774 // Memoize the node unless it returns a flag.
6775 MemIntrinsicSDNode *N;
6776 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
6777 FoldingSetNodeID ID;
6778 AddNodeIDNode(ID, Opcode, VTList, Ops);
6779 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
6780 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
6781 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6782 void *IP = nullptr;
6783 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6784 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
6785 return SDValue(E, 0);
6786 }
6787
6788 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6789 VTList, MemVT, MMO);
6790 createOperands(N, Ops);
6791
6792 CSEMap.InsertNode(N, IP);
6793 } else {
6794 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
6795 VTList, MemVT, MMO);
6796 createOperands(N, Ops);
6797 }
6798 InsertNode(N);
6799 return SDValue(N, 0);
6800}
6801
6802SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
6803 SDValue Chain, int FrameIndex,
6804 int64_t Size, int64_t Offset) {
6805 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
6806 const auto VTs = getVTList(MVT::Other);
6807 SDValue Ops[2] = {
6808 Chain,
6809 getFrameIndex(FrameIndex,
6810 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
6811 true)};
6812
6813 FoldingSetNodeID ID;
6814 AddNodeIDNode(ID, Opcode, VTs, Ops);
6815 ID.AddInteger(FrameIndex);
6816 ID.AddInteger(Size);
6817 ID.AddInteger(Offset);
6818 void *IP = nullptr;
6819 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
6820 return SDValue(E, 0);
6821
6822 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
6823 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
6824 createOperands(N, Ops);
6825 CSEMap.InsertNode(N, IP);
6826 InsertNode(N);
6827 SDValue V(N, 0);
6828 NewSDValueDbgMsg(V, "Creating new node: ", this);
6829 return V;
6830}
6831
6832/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6833/// MachinePointerInfo record from it. This is particularly useful because the
6834/// code generator has many cases where it doesn't bother passing in a
6835/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6836static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6837 SelectionDAG &DAG, SDValue Ptr,
6838 int64_t Offset = 0) {
6839 // If this is FI+Offset, we can model it.
6840 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
6841 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
6842 FI->getIndex(), Offset);
6843
6844 // If this is (FI+Offset1)+Offset2, we can model it.
6845 if (Ptr.getOpcode() != ISD::ADD ||
6846 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
6847 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
6848 return Info;
6849
6850 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6851 return MachinePointerInfo::getFixedStack(
6852 DAG.getMachineFunction(), FI,
6853 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
6854}
6855
6856/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
6857/// MachinePointerInfo record from it. This is particularly useful because the
6858/// code generator has many cases where it doesn't bother passing in a
6859/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
6860static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
6861 SelectionDAG &DAG, SDValue Ptr,
6862 SDValue OffsetOp) {
6863 // If the 'Offset' value isn't a constant, we can't handle this.
6864 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
6865 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
6866 if (OffsetOp.isUndef())
6867 return InferPointerInfo(Info, DAG, Ptr);
6868 return Info;
6869}
6870
6871SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6872 EVT VT, const SDLoc &dl, SDValue Chain,
6873 SDValue Ptr, SDValue Offset,
6874 MachinePointerInfo PtrInfo, EVT MemVT,
6875 unsigned Alignment,
6876 MachineMemOperand::Flags MMOFlags,
6877 const AAMDNodes &AAInfo, const MDNode *Ranges) {
6878 assert(Chain.getValueType() == MVT::Other &&
6879 "Invalid chain type");
6880 if (Alignment == 0) // Ensure that codegen never sees alignment 0
6881 Alignment = getEVTAlignment(MemVT);
6882
6883 MMOFlags |= MachineMemOperand::MOLoad;
6884 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
6885 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
6886 // clients.
6887 if (PtrInfo.V.isNull())
6888 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
6889
6890 MachineFunction &MF = getMachineFunction();
6891 MachineMemOperand *MMO = MF.getMachineMemOperand(
6892 PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges);
6893 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
6894}
6895
6896SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
6897 EVT VT, const SDLoc &dl, SDValue Chain,
6898 SDValue Ptr, SDValue Offset, EVT MemVT,
6899 MachineMemOperand *MMO) {
6900 if (VT == MemVT) {
6901 ExtType = ISD::NON_EXTLOAD;
6902 } else if (ExtType == ISD::NON_EXTLOAD) {
6903 assert(VT == MemVT && "Non-extending load from different memory type!");
6904 } else {
6905 // Extending load.
6906 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
6907 "Should only be an extending load, not truncating!");
6908 assert(VT.isInteger() == MemVT.isInteger() &&
6909 "Cannot convert from FP to Int or Int -> FP!");
6910 assert(VT.isVector() == MemVT.isVector() &&
6911 "Cannot use an ext load to convert to or from a vector!");
6912 assert((!VT.isVector() ||
6913 VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
6914 "Cannot use an ext load to change the number of vector elements!");
6915 }
6916
6917 bool Indexed = AM != ISD::UNINDEXED;
6918 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
6919
6920 SDVTList VTs = Indexed ?
6921 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
6922 SDValue Ops[] = { Chain, Ptr, Offset };
6923 FoldingSetNodeID ID;
6924 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
6925 ID.AddInteger(MemVT.getRawBits());
6926 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
6927 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
6928 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6929 void *IP = nullptr;
6930 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
6931 cast<LoadSDNode>(E)->refineAlignment(MMO);
6932 return SDValue(E, 0);
6933 }
6934 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
6935 ExtType, MemVT, MMO);
6936 createOperands(N, Ops);
6937
6938 CSEMap.InsertNode(N, IP);
6939 InsertNode(N);
6940 SDValue V(N, 0);
6941 NewSDValueDbgMsg(V, "Creating new node: ", this);
6942 return V;
6943}
6944
6945SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6946 SDValue Ptr, MachinePointerInfo PtrInfo,
6947 unsigned Alignment,
6948 MachineMemOperand::Flags MMOFlags,
6949 const AAMDNodes &AAInfo, const MDNode *Ranges) {
6950 SDValue Undef = getUNDEF(Ptr.getValueType());
6951 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
6952 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
6953}
6954
6955SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6956 SDValue Ptr, MachineMemOperand *MMO) {
6957 SDValue Undef = getUNDEF(Ptr.getValueType());
6958 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
6959 VT, MMO);
6960}
6961
6962SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6963 EVT VT, SDValue Chain, SDValue Ptr,
6964 MachinePointerInfo PtrInfo, EVT MemVT,
6965 unsigned Alignment,
6966 MachineMemOperand::Flags MMOFlags,
6967 const AAMDNodes &AAInfo) {
6968 SDValue Undef = getUNDEF(Ptr.getValueType());
6969 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
6970 MemVT, Alignment, MMOFlags, AAInfo);
6971}
6972
6973SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
6974 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
6975 MachineMemOperand *MMO) {
6976 SDValue Undef = getUNDEF(Ptr.getValueType());
6977 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
6978 MemVT, MMO);
6979}
6980
6981SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
6982 SDValue Base, SDValue Offset,
6983 ISD::MemIndexedMode AM) {
6984 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
6985 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
6986 // Don't propagate the invariant or dereferenceable flags.
6987 auto MMOFlags =
6988 LD->getMemOperand()->getFlags() &
6989 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
6990 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
6991 LD->getChain(), Base, Offset, LD->getPointerInfo(),
6992 LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
6993 LD->getAAInfo());
6994}
6995
6996SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
6997 SDValue Ptr, MachinePointerInfo PtrInfo,
6998 unsigned Alignment,
6999 MachineMemOperand::Flags MMOFlags,
7000 const AAMDNodes &AAInfo) {
7001 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
7002 if (Alignment == 0) // Ensure that codegen never sees alignment 0
7003 Alignment = getEVTAlignment(Val.getValueType());
7004
7005 MMOFlags |= MachineMemOperand::MOStore;
7006 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
7007
7008 if (PtrInfo.V.isNull())
7009 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7010
7011 MachineFunction &MF = getMachineFunction();
7012 MachineMemOperand *MMO = MF.getMachineMemOperand(
7013 PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo);
7014 return getStore(Chain, dl, Val, Ptr, MMO);
7015}
7016
7017SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7018 SDValue Ptr, MachineMemOperand *MMO) {
7019 assert(Chain.getValueType() == MVT::Other &&
7020 "Invalid chain type");
7021 EVT VT = Val.getValueType();
7022 SDVTList VTs = getVTList(MVT::Other);
7023 SDValue Undef = getUNDEF(Ptr.getValueType());
7024 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7025 FoldingSetNodeID ID;
7026 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7027 ID.AddInteger(VT.getRawBits());
7028 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7029 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
7030 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7031 void *IP = nullptr;
7032 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7033 cast<StoreSDNode>(E)->refineAlignment(MMO);
7034 return SDValue(E, 0);
7035 }
7036 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7037 ISD::UNINDEXED, false, VT, MMO);
7038 createOperands(N, Ops);
7039
7040 CSEMap.InsertNode(N, IP);
7041 InsertNode(N);
7042 SDValue V(N, 0);
7043 NewSDValueDbgMsg(V, "Creating new node: ", this);
7044 return V;
7045}
7046
7047SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7048 SDValue Ptr, MachinePointerInfo PtrInfo,
7049 EVT SVT, unsigned Alignment,
7050 MachineMemOperand::Flags MMOFlags,
7051 const AAMDNodes &AAInfo) {
7052 assert(Chain.getValueType() == MVT::Other &&
7053 "Invalid chain type");
7054 if (Alignment == 0) // Ensure that codegen never sees alignment 0
7055 Alignment = getEVTAlignment(SVT);
7056
7057 MMOFlags |= MachineMemOperand::MOStore;
7058 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
7059
7060 if (PtrInfo.V.isNull())
7061 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
7062
7063 MachineFunction &MF = getMachineFunction();
7064 MachineMemOperand *MMO = MF.getMachineMemOperand(
7065 PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
7066 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
7067}
7068
7069SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
7070 SDValue Ptr, EVT SVT,
7071 MachineMemOperand *MMO) {
7072 EVT VT = Val.getValueType();
7073
7074 assert(Chain.getValueType() == MVT::Other &&
7075 "Invalid chain type");
7076 if (VT == SVT)
7077 return getStore(Chain, dl, Val, Ptr, MMO);
7078
7079 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
7080 "Should only be a truncating store, not extending!");
7081 assert(VT.isInteger() == SVT.isInteger() &&
7082 "Can't do FP-INT conversion!");
7083 assert(VT.isVector() == SVT.isVector() &&
7084 "Cannot use trunc store to convert to or from a vector!");
7085 assert((!VT.isVector() ||
7086 VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
7087 "Cannot use trunc store to change the number of vector elements!");
7088
7089 SDVTList VTs = getVTList(MVT::Other);
7090 SDValue Undef = getUNDEF(Ptr.getValueType());
7091 SDValue Ops[] = { Chain, Val, Ptr, Undef };
7092 FoldingSetNodeID ID;
7093 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7094 ID.AddInteger(SVT.getRawBits());
7095 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
7096 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
7097 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7098 void *IP = nullptr;
7099 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7100 cast<StoreSDNode>(E)->refineAlignment(MMO);
7101 return SDValue(E, 0);
7102 }
7103 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7104 ISD::UNINDEXED, true, SVT, MMO);
7105 createOperands(N, Ops);
7106
7107 CSEMap.InsertNode(N, IP);
7108 InsertNode(N);
7109 SDValue V(N, 0);
7110 NewSDValueDbgMsg(V, "Creating new node: ", this);
7111 return V;
7112}
7113
7114SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
7115 SDValue Base, SDValue Offset,
7116 ISD::MemIndexedMode AM) {
7117 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
7118 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
7119 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
7120 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
7121 FoldingSetNodeID ID;
7122 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
7123 ID.AddInteger(ST->getMemoryVT().getRawBits());
7124 ID.AddInteger(ST->getRawSubclassData());
7125 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
7126 void *IP = nullptr;
7127 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
7128 return SDValue(E, 0);
7129
7130 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
7131 ST->isTruncatingStore(), ST->getMemoryVT(),
7132 ST->getMemOperand());
7133 createOperands(N, Ops);
7134
7135 CSEMap.InsertNode(N, IP);
7136 InsertNode(N);
7137 SDValue V(N, 0);
7138 NewSDValueDbgMsg(V, "Creating new node: ", this);
7139 return V;
7140}
7141
7142SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
7143 SDValue Ptr, SDValue Mask, SDValue PassThru,
7144 EVT MemVT, MachineMemOperand *MMO,
7145 ISD::LoadExtType ExtTy, bool isExpanding) {
7146 SDVTList VTs = getVTList(VT, MVT::Other);
7147 SDValue Ops[] = { Chain, Ptr, Mask, PassThru };
7148 FoldingSetNodeID ID;
7149 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
7150 ID.AddInteger(VT.getRawBits());
7151 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
7152 dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
7153 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7154 void *IP = nullptr;
7155 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7156 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
7157 return SDValue(E, 0);
7158 }
7159 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7160 ExtTy, isExpanding, MemVT, MMO);
7161 createOperands(N, Ops);
7162
7163 CSEMap.InsertNode(N, IP);
7164 InsertNode(N);
7165 SDValue V(N, 0);
7166 NewSDValueDbgMsg(V, "Creating new node: ", this);
7167 return V;
7168}
7169
7170SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
7171 SDValue Val, SDValue Ptr, SDValue Mask,
7172 EVT MemVT, MachineMemOperand *MMO,
7173 bool IsTruncating, bool IsCompressing) {
7174 assert(Chain.getValueType() == MVT::Other &&
7175 "Invalid chain type");
7176 EVT VT = Val.getValueType();
7177 SDVTList VTs = getVTList(MVT::Other);
7178 SDValue Ops[] = { Chain, Val, Ptr, Mask };
7179 FoldingSetNodeID ID;
7180 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
7181 ID.AddInteger(VT.getRawBits());
7182 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
7183 dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
7184 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7185 void *IP = nullptr;
7186 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7187 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
7188 return SDValue(E, 0);
7189 }
7190 auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
7191 IsTruncating, IsCompressing, MemVT, MMO);
7192 createOperands(N, Ops);
7193
7194 CSEMap.InsertNode(N, IP);
7195 InsertNode(N);
7196 SDValue V(N, 0);
7197 NewSDValueDbgMsg(V, "Creating new node: ", this);
7198 return V;
7199}
7200
7201SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
7202 ArrayRef<SDValue> Ops,
7203 MachineMemOperand *MMO) {
7204 assert(Ops.size() == 6 && "Incompatible number of operands");
7205
7206 FoldingSetNodeID ID;
7207 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
7208 ID.AddInteger(VT.getRawBits());
7209 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
7210 dl.getIROrder(), VTs, VT, MMO));
7211 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7212 void *IP = nullptr;
7213 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7214 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
7215 return SDValue(E, 0);
7216 }
7217
7218 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7219 VTs, VT, MMO);
7220 createOperands(N, Ops);
7221
7222 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
7223 "Incompatible type of the PassThru value in MaskedGatherSDNode");
7224 assert(N->getMask().getValueType().getVectorNumElements() ==
7225 N->getValueType(0).getVectorNumElements() &&
7226 "Vector width mismatch between mask and data");
7227 assert(N->getIndex().getValueType().getVectorNumElements() >=
7228 N->getValueType(0).getVectorNumElements() &&
7229 "Vector width mismatch between index and data");
7230 assert(isa<ConstantSDNode>(N->getScale()) &&
7231 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
7232 "Scale should be a constant power of 2");
7233
7234 CSEMap.InsertNode(N, IP);
7235 InsertNode(N);
7236 SDValue V(N, 0);
7237 NewSDValueDbgMsg(V, "Creating new node: ", this);
7238 return V;
7239}
7240
7241SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
7242 ArrayRef<SDValue> Ops,
7243 MachineMemOperand *MMO) {
7244 assert(Ops.size() == 6 && "Incompatible number of operands");
7245
7246 FoldingSetNodeID ID;
7247 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
7248 ID.AddInteger(VT.getRawBits());
7249 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
7250 dl.getIROrder(), VTs, VT, MMO));
7251 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
7252 void *IP = nullptr;
7253 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
7254 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
7255 return SDValue(E, 0);
7256 }
7257 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
7258 VTs, VT, MMO);
7259 createOperands(N, Ops);
7260
7261 assert(N->getMask().getValueType().getVectorNumElements() ==
7262 N->getValue().getValueType().getVectorNumElements() &&
7263 "Vector width mismatch between mask and data");
7264 assert(N->getIndex().getValueType().getVectorNumElements() >=
7265 N->getValue().getValueType().getVectorNumElements() &&
7266 "Vector width mismatch between index and data");
7267 assert(isa<ConstantSDNode>(N->getScale()) &&
7268 cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
7269 "Scale should be a constant power of 2");
7270
7271 CSEMap.InsertNode(N, IP);
7272 InsertNode(N);
7273 SDValue V(N, 0);
7274 NewSDValueDbgMsg(V, "Creating new node: ", this);
7275 return V;
7276}
7277
7278SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
7279 // select undef, T, F --> T (if T is a constant), otherwise F
7280 // select, ?, undef, F --> F
7281 // select, ?, T, undef --> T
7282 if (Cond.isUndef())
7283 return isConstantValueOfAnyType(T) ? T : F;
7284 if (T.isUndef())
7285 return F;
7286 if (F.isUndef())
7287 return T;
7288
7289 // select true, T, F --> T
7290 // select false, T, F --> F
7291 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
7292 return CondC->isNullValue() ? F : T;
7293
7294 // TODO: This should simplify VSELECT with constant condition using something
7295 // like this (but check boolean contents to be complete?):
7296 // if (ISD::isBuildVectorAllOnes(Cond.getNode()))
7297 // return T;
7298 // if (ISD::isBuildVectorAllZeros(Cond.getNode()))
7299 // return F;
7300
7301 // select ?, T, T --> T
7302 if (T == F)
7303 return T;
7304
7305 return SDValue();
7306}
7307
7308SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
7309 // shift undef, Y --> 0 (can always assume that the undef value is 0)
7310 if (X.isUndef())
7311 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
7312 // shift X, undef --> undef (because it may shift by the bitwidth)
7313 if (Y.isUndef())
7314 return getUNDEF(X.getValueType());
7315
7316 // shift 0, Y --> 0
7317 // shift X, 0 --> X
7318 if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
7319 return X;
7320
7321 // shift X, C >= bitwidth(X) --> undef
7322 // All vector elements must be too big (or undef) to avoid partial undefs.
7323 auto isShiftTooBig = [X](ConstantSDNode *Val) {
7324 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
7325 };
7326 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
7327 return getUNDEF(X.getValueType());
7328
7329 return SDValue();
7330}
7331
7332// TODO: Use fast-math-flags to enable more simplifications.
7333SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y) {
7334 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
7335 if (!YC)
7336 return SDValue();
7337
7338 // X + -0.0 --> X
7339 if (Opcode == ISD::FADD)
7340 if (YC->getValueAPF().isNegZero())
7341 return X;
7342
7343 // X - +0.0 --> X
7344 if (Opcode == ISD::FSUB)
7345 if (YC->getValueAPF().isPosZero())
7346 return X;
7347
7348 // X * 1.0 --> X
7349 // X / 1.0 --> X
7350 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
7351 if (YC->getValueAPF().isExactlyValue(1.0))
7352 return X;
7353
7354 return SDValue();
7355}
7356
7357SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
7358 SDValue Ptr, SDValue SV, unsigned Align) {
7359 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
7360 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
7361}
7362
7363SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7364 ArrayRef<SDUse> Ops) {
7365 switch (Ops.size()) {
7366 case 0: return getNode(Opcode, DL, VT);
7367 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
7368 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
7369 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
7370 default: break;
7371 }
7372
7373 // Copy from an SDUse array into an SDValue array for use with
7374 // the regular getNode logic.
7375 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
7376 return getNode(Opcode, DL, VT, NewOps);
7377}
7378
7379SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7380 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
7381 unsigned NumOps = Ops.size();
7382 switch (NumOps) {
7383 case 0: return getNode(Opcode, DL, VT);
7384 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
7385 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
7386 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
7387 default: break;
7388 }
7389
7390 switch (Opcode) {
7391 default: break;
7392 case ISD::BUILD_VECTOR:
7393 // Attempt to simplify BUILD_VECTOR.
7394 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7395 return V;
7396 break;
7397 case ISD::CONCAT_VECTORS:
7398 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7399 return V;
7400 break;
7401 case ISD::SELECT_CC:
7402 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
7403 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
7404 "LHS and RHS of condition must have same type!");
7405 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
7406 "True and False arms of SelectCC must have same type!");
7407 assert(Ops[2].getValueType() == VT &&
7408 "select_cc node must be of same type as true and false value!");
7409 break;
7410 case ISD::BR_CC:
7411 assert(NumOps == 5 && "BR_CC takes 5 operands!");
7412 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
7413 "LHS/RHS of comparison should match types!");
7414 break;
7415 }
7416
7417 // Memoize nodes.
7418 SDNode *N;
7419 SDVTList VTs = getVTList(VT);
7420
7421 if (VT != MVT::Glue) {
7422 FoldingSetNodeID ID;
7423 AddNodeIDNode(ID, Opcode, VTs, Ops);
7424 void *IP = nullptr;
7425
7426 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7427 return SDValue(E, 0);
7428
7429 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7430 createOperands(N, Ops);
7431
7432 CSEMap.InsertNode(N, IP);
7433 } else {
7434 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7435 createOperands(N, Ops);
7436 }
7437
7438 InsertNode(N);
7439 SDValue V(N, 0);
7440 NewSDValueDbgMsg(V, "Creating new node: ", this);
7441 return V;
7442}
7443
7444SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7445 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
7446 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
7447}
7448
7449SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7450 ArrayRef<SDValue> Ops) {
7451 if (VTList.NumVTs == 1)
7452 return getNode(Opcode, DL, VTList.VTs[0], Ops);
7453
7454#if 0
7455 switch (Opcode) {
7456 // FIXME: figure out how to safely handle things like
7457 // int foo(int x) { return 1 << (x & 255); }
7458 // int bar() { return foo(256); }
7459 case ISD::SRA_PARTS:
7460 case ISD::SRL_PARTS:
7461 case ISD::SHL_PARTS:
7462 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
7463 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
7464 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
7465 else if (N3.getOpcode() == ISD::AND)
7466 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
7467 // If the and is only masking out bits that cannot effect the shift,
7468 // eliminate the and.
7469 unsigned NumBits = VT.getScalarSizeInBits()*2;
7470 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
7471 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
7472 }
7473 break;
7474 }
7475#endif
7476
7477 // Memoize the node unless it returns a flag.
7478 SDNode *N;
7479 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
7480 FoldingSetNodeID ID;
7481 AddNodeIDNode(ID, Opcode, VTList, Ops);
7482 void *IP = nullptr;
7483 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
7484 return SDValue(E, 0);
7485
7486 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
7487 createOperands(N, Ops);
7488 CSEMap.InsertNode(N, IP);
7489 } else {
7490 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
7491 createOperands(N, Ops);
7492 }
7493 InsertNode(N);
7494 SDValue V(N, 0);
7495 NewSDValueDbgMsg(V, "Creating new node: ", this);
7496 return V;
7497}
7498
7499SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
7500 SDVTList VTList) {
7501 return getNode(Opcode, DL, VTList, None);
7502}
7503
7504SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7505 SDValue N1) {
7506 SDValue Ops[] = { N1 };
7507 return getNode(Opcode, DL, VTList, Ops);
7508}
7509
7510SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7511 SDValue N1, SDValue N2) {
7512 SDValue Ops[] = { N1, N2 };
7513 return getNode(Opcode, DL, VTList, Ops);
7514}
7515
7516SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7517 SDValue N1, SDValue N2, SDValue N3) {
7518 SDValue Ops[] = { N1, N2, N3 };
7519 return getNode(Opcode, DL, VTList, Ops);
7520}
7521
7522SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7523 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7524 SDValue Ops[] = { N1, N2, N3, N4 };
7525 return getNode(Opcode, DL, VTList, Ops);
7526}
7527
7528SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
7529 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7530 SDValue N5) {
7531 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7532 return getNode(Opcode, DL, VTList, Ops);
7533}
7534
7535SDVTList SelectionDAG::getVTList(EVT VT) {
7536 return makeVTList(SDNode::getValueTypeList(VT), 1);
7537}
7538
7539SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
7540 FoldingSetNodeID ID;
7541 ID.AddInteger(2U);
7542 ID.AddInteger(VT1.getRawBits());
7543 ID.AddInteger(VT2.getRawBits());
7544
7545 void *IP = nullptr;
7546 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7547 if (!Result) {
7548 EVT *Array = Allocator.Allocate<EVT>(2);
7549 Array[0] = VT1;
7550 Array[1] = VT2;
7551 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
7552 VTListMap.InsertNode(Result, IP);
7553 }
7554 return Result->getSDVTList();
7555}
7556
7557SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
7558 FoldingSetNodeID ID;
7559 ID.AddInteger(3U);
7560 ID.AddInteger(VT1.getRawBits());
7561 ID.AddInteger(VT2.getRawBits());
7562 ID.AddInteger(VT3.getRawBits());
7563
7564 void *IP = nullptr;
7565 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7566 if (!Result) {
7567 EVT *Array = Allocator.Allocate<EVT>(3);
7568 Array[0] = VT1;
7569 Array[1] = VT2;
7570 Array[2] = VT3;
7571 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
7572 VTListMap.InsertNode(Result, IP);
7573 }
7574 return Result->getSDVTList();
7575}
7576
7577SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
7578 FoldingSetNodeID ID;
7579 ID.AddInteger(4U);
7580 ID.AddInteger(VT1.getRawBits());
7581 ID.AddInteger(VT2.getRawBits());
7582 ID.AddInteger(VT3.getRawBits());
7583 ID.AddInteger(VT4.getRawBits());
7584
7585 void *IP = nullptr;
7586 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7587 if (!Result) {
7588 EVT *Array = Allocator.Allocate<EVT>(4);
7589 Array[0] = VT1;
7590 Array[1] = VT2;
7591 Array[2] = VT3;
7592 Array[3] = VT4;
7593 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
7594 VTListMap.InsertNode(Result, IP);
7595 }
7596 return Result->getSDVTList();
7597}
7598
7599SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
7600 unsigned NumVTs = VTs.size();
7601 FoldingSetNodeID ID;
7602 ID.AddInteger(NumVTs);
7603 for (unsigned index = 0; index < NumVTs; index++) {
7604 ID.AddInteger(VTs[index].getRawBits());
7605 }
7606
7607 void *IP = nullptr;
7608 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
7609 if (!Result) {
7610 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
7611 llvm::copy(VTs, Array);
7612 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
7613 VTListMap.InsertNode(Result, IP);
7614 }
7615 return Result->getSDVTList();
7616}
7617
7618
7619/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
7620/// specified operands. If the resultant node already exists in the DAG,
7621/// this does not modify the specified node, instead it returns the node that
7622/// already exists. If the resultant node does not exist in the DAG, the
7623/// input node is returned. As a degenerate case, if you specify the same
7624/// input operands as the node already has, the input node is returned.
7625SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
7626 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
7627
7628 // Check to see if there is no change.
7629 if (Op == N->getOperand(0)) return N;
7630
7631 // See if the modified node already exists.
7632 void *InsertPos = nullptr;
7633 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
7634 return Existing;
7635
7636 // Nope it doesn't. Remove the node from its current place in the maps.
7637 if (InsertPos)
7638 if (!RemoveNodeFromCSEMaps(N))
7639 InsertPos = nullptr;
7640
7641 // Now we update the operands.
7642 N->OperandList[0].set(Op);
7643
7644 updateDivergence(N);
7645 // If this gets put into a CSE map, add it.
7646 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7647 return N;
7648}
7649
7650SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
7651 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
7652
7653 // Check to see if there is no change.
7654 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
7655 return N; // No operands changed, just return the input node.
7656
7657 // See if the modified node already exists.
7658 void *InsertPos = nullptr;
7659 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
7660 return Existing;
7661
7662 // Nope it doesn't. Remove the node from its current place in the maps.
7663 if (InsertPos)
7664 if (!RemoveNodeFromCSEMaps(N))
7665 InsertPos = nullptr;
7666
7667 // Now we update the operands.
7668 if (N->OperandList[0] != Op1)
7669 N->OperandList[0].set(Op1);
7670 if (N->OperandList[1] != Op2)
7671 N->OperandList[1].set(Op2);
7672
7673 updateDivergence(N);
7674 // If this gets put into a CSE map, add it.
7675 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7676 return N;
7677}
7678
7679SDNode *SelectionDAG::
7680UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
7681 SDValue Ops[] = { Op1, Op2, Op3 };
7682 return UpdateNodeOperands(N, Ops);
7683}
7684
7685SDNode *SelectionDAG::
7686UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7687 SDValue Op3, SDValue Op4) {
7688 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
7689 return UpdateNodeOperands(N, Ops);
7690}
7691
7692SDNode *SelectionDAG::
7693UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
7694 SDValue Op3, SDValue Op4, SDValue Op5) {
7695 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
7696 return UpdateNodeOperands(N, Ops);
7697}
7698
7699SDNode *SelectionDAG::
7700UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
7701 unsigned NumOps = Ops.size();
7702 assert(N->getNumOperands() == NumOps &&
7703 "Update with wrong number of operands");
7704
7705 // If no operands changed just return the input node.
7706 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
7707 return N;
7708
7709 // See if the modified node already exists.
7710 void *InsertPos = nullptr;
7711 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
7712 return Existing;
7713
7714 // Nope it doesn't. Remove the node from its current place in the maps.
7715 if (InsertPos)
7716 if (!RemoveNodeFromCSEMaps(N))
7717 InsertPos = nullptr;
7718
7719 // Now we update the operands.
7720 for (unsigned i = 0; i != NumOps; ++i)
7721 if (N->OperandList[i] != Ops[i])
7722 N->OperandList[i].set(Ops[i]);
7723
7724 updateDivergence(N);
7725 // If this gets put into a CSE map, add it.
7726 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
7727 return N;
7728}
7729
7730/// DropOperands - Release the operands and set this node to have
7731/// zero operands.
7732void SDNode::DropOperands() {
7733 // Unlike the code in MorphNodeTo that does this, we don't need to
7734 // watch for dead nodes here.
7735 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
7736 SDUse &Use = *I++;
7737 Use.set(SDValue());
7738 }
7739}
7740
7741void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
7742 ArrayRef<MachineMemOperand *> NewMemRefs) {
7743 if (NewMemRefs.empty()) {
7744 N->clearMemRefs();
7745 return;
7746 }
7747
7748 // Check if we can avoid allocating by storing a single reference directly.
7749 if (NewMemRefs.size() == 1) {
7750 N->MemRefs = NewMemRefs[0];
7751 N->NumMemRefs = 1;
7752 return;
7753 }
7754
7755 MachineMemOperand **MemRefsBuffer =
7756 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
7757 llvm::copy(NewMemRefs, MemRefsBuffer);
7758 N->MemRefs = MemRefsBuffer;
7759 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
7760}
7761
7762/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
7763/// machine opcode.
7764///
7765SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7766 EVT VT) {
7767 SDVTList VTs = getVTList(VT);
7768 return SelectNodeTo(N, MachineOpc, VTs, None);
7769}
7770
7771SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7772 EVT VT, SDValue Op1) {
7773 SDVTList VTs = getVTList(VT);
7774 SDValue Ops[] = { Op1 };
7775 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7776}
7777
7778SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7779 EVT VT, SDValue Op1,
7780 SDValue Op2) {
7781 SDVTList VTs = getVTList(VT);
7782 SDValue Ops[] = { Op1, Op2 };
7783 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7784}
7785
7786SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7787 EVT VT, SDValue Op1,
7788 SDValue Op2, SDValue Op3) {
7789 SDVTList VTs = getVTList(VT);
7790 SDValue Ops[] = { Op1, Op2, Op3 };
7791 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7792}
7793
7794SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7795 EVT VT, ArrayRef<SDValue> Ops) {
7796 SDVTList VTs = getVTList(VT);
7797 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7798}
7799
7800SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7801 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
7802 SDVTList VTs = getVTList(VT1, VT2);
7803 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7804}
7805
7806SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7807 EVT VT1, EVT VT2) {
7808 SDVTList VTs = getVTList(VT1, VT2);
7809 return SelectNodeTo(N, MachineOpc, VTs, None);
7810}
7811
7812SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7813 EVT VT1, EVT VT2, EVT VT3,
7814 ArrayRef<SDValue> Ops) {
7815 SDVTList VTs = getVTList(VT1, VT2, VT3);
7816 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7817}
7818
7819SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7820 EVT VT1, EVT VT2,
7821 SDValue Op1, SDValue Op2) {
7822 SDVTList VTs = getVTList(VT1, VT2);
7823 SDValue Ops[] = { Op1, Op2 };
7824 return SelectNodeTo(N, MachineOpc, VTs, Ops);
7825}
7826
7827SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
7828 SDVTList VTs,ArrayRef<SDValue> Ops) {
7829 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
7830 // Reset the NodeID to -1.
7831 New->setNodeId(-1);
7832 if (New != N) {
7833 ReplaceAllUsesWith(N, New);
7834 RemoveDeadNode(N);
7835 }
7836 return New;
7837}
7838
7839/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
7840/// the line number information on the merged node since it is not possible to
7841/// preserve the information that operation is associated with multiple lines.
7842/// This will make the debugger working better at -O0, were there is a higher
7843/// probability having other instructions associated with that line.
7844///
7845/// For IROrder, we keep the smaller of the two
7846SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
7847 DebugLoc NLoc = N->getDebugLoc();
7848 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
7849 N->setDebugLoc(DebugLoc());
7850 }
7851 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
7852 N->setIROrder(Order);
7853 return N;
7854}
7855
7856/// MorphNodeTo - This *mutates* the specified node to have the specified
7857/// return type, opcode, and operands.
7858///
7859/// Note that MorphNodeTo returns the resultant node. If there is already a
7860/// node of the specified opcode and operands, it returns that node instead of
7861/// the current one. Note that the SDLoc need not be the same.
7862///
7863/// Using MorphNodeTo is faster than creating a new node and swapping it in
7864/// with ReplaceAllUsesWith both because it often avoids allocating a new
7865/// node, and because it doesn't require CSE recalculation for any of
7866/// the node's users.
7867///
7868/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
7869/// As a consequence it isn't appropriate to use from within the DAG combiner or
7870/// the legalizer which maintain worklists that would need to be updated when
7871/// deleting things.
7872SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
7873 SDVTList VTs, ArrayRef<SDValue> Ops) {
7874 // If an identical node already exists, use it.
7875 void *IP = nullptr;
7876 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
7877 FoldingSetNodeID ID;
7878 AddNodeIDNode(ID, Opc, VTs, Ops);
7879 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
7880 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
7881 }
7882
7883 if (!RemoveNodeFromCSEMaps(N))
7884 IP = nullptr;
7885
7886 // Start the morphing.
7887 N->NodeType = Opc;
7888 N->ValueList = VTs.VTs;
7889 N->NumValues = VTs.NumVTs;
7890
7891 // Clear the operands list, updating used nodes to remove this from their
7892 // use list. Keep track of any operands that become dead as a result.
7893 SmallPtrSet<SDNode*, 16> DeadNodeSet;
7894 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
7895 SDUse &Use = *I++;
7896 SDNode *Used = Use.getNode();
7897 Use.set(SDValue());
7898 if (Used->use_empty())
7899 DeadNodeSet.insert(Used);
7900 }
7901
7902 // For MachineNode, initialize the memory references information.
7903 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
7904 MN->clearMemRefs();
7905
7906 // Swap for an appropriately sized array from the recycler.
7907 removeOperands(N);
7908 createOperands(N, Ops);
7909
7910 // Delete any nodes that are still dead after adding the uses for the
7911 // new operands.
7912 if (!DeadNodeSet.empty()) {
7913 SmallVector<SDNode *, 16> DeadNodes;
7914 for (SDNode *N : DeadNodeSet)
7915 if (N->use_empty())
7916 DeadNodes.push_back(N);
7917 RemoveDeadNodes(DeadNodes);
7918 }
7919
7920 if (IP)
7921 CSEMap.InsertNode(N, IP); // Memoize the new node.
7922 return N;
7923}
7924
7925SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
7926 unsigned OrigOpc = Node->getOpcode();
7927 unsigned NewOpc;
7928 switch (OrigOpc) {
7929 default:
7930 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
7931 case ISD::STRICT_FADD: NewOpc = ISD::FADD; break;
7932 case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break;
7933 case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break;
7934 case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break;
7935 case ISD::STRICT_FREM: NewOpc = ISD::FREM; break;
7936 case ISD::STRICT_FMA: NewOpc = ISD::FMA; break;
7937 case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; break;
7938 case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break;
7939 case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break;
7940 case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; break;
7941 case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; break;
7942 case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; break;
7943 case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; break;
7944 case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; break;
7945 case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; break;
7946 case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; break;
7947 case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; break;
7948 case ISD::STRICT_FNEARBYINT: NewOpc = ISD::FNEARBYINT; break;
7949 case ISD::STRICT_FMAXNUM: NewOpc = ISD::FMAXNUM; break;
7950 case ISD::STRICT_FMINNUM: NewOpc = ISD::FMINNUM; break;
7951 case ISD::STRICT_FCEIL: NewOpc = ISD::FCEIL; break;
7952 case ISD::STRICT_FFLOOR: NewOpc = ISD::FFLOOR; break;
7953 case ISD::STRICT_FROUND: NewOpc = ISD::FROUND; break;
7954 case ISD::STRICT_FTRUNC: NewOpc = ISD::FTRUNC; break;
7955 case ISD::STRICT_FP_ROUND: NewOpc = ISD::FP_ROUND; break;
7956 case ISD::STRICT_FP_EXTEND: NewOpc = ISD::FP_EXTEND; break;
7957 }
7958
7959 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
7960
7961 // We're taking this node out of the chain, so we need to re-link things.
7962 SDValue InputChain = Node->getOperand(0);
7963 SDValue OutputChain = SDValue(Node, 1);
7964 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
7965
7966 SmallVector<SDValue, 3> Ops;
7967 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
7968 Ops.push_back(Node->getOperand(i));
7969
7970 SDVTList VTs = getVTList(Node->getValueType(0));
7971 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
7972
7973 // MorphNodeTo can operate in two ways: if an existing node with the
7974 // specified operands exists, it can just return it. Otherwise, it
7975 // updates the node in place to have the requested operands.
7976 if (Res == Node) {
7977 // If we updated the node in place, reset the node ID. To the isel,
7978 // this should be just like a newly allocated machine node.
7979 Res->setNodeId(-1);
7980 } else {
7981 ReplaceAllUsesWith(Node, Res);
7982 RemoveDeadNode(Node);
7983 }
7984
7985 return Res;
7986}
7987
7988/// getMachineNode - These are used for target selectors to create a new node
7989/// with specified return type(s), MachineInstr opcode, and operands.
7990///
7991/// Note that getMachineNode returns the resultant node. If there is already a
7992/// node of the specified opcode and operands, it returns that node instead of
7993/// the current one.
7994MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
7995 EVT VT) {
7996 SDVTList VTs = getVTList(VT);
7997 return getMachineNode(Opcode, dl, VTs, None);
7998}
7999
8000MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8001 EVT VT, SDValue Op1) {
8002 SDVTList VTs = getVTList(VT);
8003 SDValue Ops[] = { Op1 };
8004 return getMachineNode(Opcode, dl, VTs, Ops);
8005}
8006
8007MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8008 EVT VT, SDValue Op1, SDValue Op2) {
8009 SDVTList VTs = getVTList(VT);
8010 SDValue Ops[] = { Op1, Op2 };
8011 return getMachineNode(Opcode, dl, VTs, Ops);
8012}
8013
8014MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8015 EVT VT, SDValue Op1, SDValue Op2,
8016 SDValue Op3) {
8017 SDVTList VTs = getVTList(VT);
8018 SDValue Ops[] = { Op1, Op2, Op3 };
8019 return getMachineNode(Opcode, dl, VTs, Ops);
8020}
8021
8022MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8023 EVT VT, ArrayRef<SDValue> Ops) {
8024 SDVTList VTs = getVTList(VT);
8025 return getMachineNode(Opcode, dl, VTs, Ops);
8026}
8027
8028MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8029 EVT VT1, EVT VT2, SDValue Op1,
8030 SDValue Op2) {
8031 SDVTList VTs = getVTList(VT1, VT2);
8032 SDValue Ops[] = { Op1, Op2 };
8033 return getMachineNode(Opcode, dl, VTs, Ops);
8034}
8035
8036MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8037 EVT VT1, EVT VT2, SDValue Op1,
8038 SDValue Op2, SDValue Op3) {
8039 SDVTList VTs = getVTList(VT1, VT2);
8040 SDValue Ops[] = { Op1, Op2, Op3 };
8041 return getMachineNode(Opcode, dl, VTs, Ops);
8042}
8043
8044MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8045 EVT VT1, EVT VT2,
8046 ArrayRef<SDValue> Ops) {
8047 SDVTList VTs = getVTList(VT1, VT2);
8048 return getMachineNode(Opcode, dl, VTs, Ops);
8049}
8050
8051MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8052 EVT VT1, EVT VT2, EVT VT3,
8053 SDValue Op1, SDValue Op2) {
8054 SDVTList VTs = getVTList(VT1, VT2, VT3);
8055 SDValue Ops[] = { Op1, Op2 };
8056 return getMachineNode(Opcode, dl, VTs, Ops);
8057}
8058
8059MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8060 EVT VT1, EVT VT2, EVT VT3,
8061 SDValue Op1, SDValue Op2,
8062 SDValue Op3) {
8063 SDVTList VTs = getVTList(VT1, VT2, VT3);
8064 SDValue Ops[] = { Op1, Op2, Op3 };
8065 return getMachineNode(Opcode, dl, VTs, Ops);
8066}
8067
8068MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8069 EVT VT1, EVT VT2, EVT VT3,
8070 ArrayRef<SDValue> Ops) {
8071 SDVTList VTs = getVTList(VT1, VT2, VT3);
8072 return getMachineNode(Opcode, dl, VTs, Ops);
8073}
8074
8075MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
8076 ArrayRef<EVT> ResultTys,
8077 ArrayRef<SDValue> Ops) {
8078 SDVTList VTs = getVTList(ResultTys);
8079 return getMachineNode(Opcode, dl, VTs, Ops);
8080}
8081
8082MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
8083 SDVTList VTs,
8084 ArrayRef<SDValue> Ops) {
8085 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
8086 MachineSDNode *N;
8087 void *IP = nullptr;
8088
8089 if (DoCSE) {
8090 FoldingSetNodeID ID;
8091 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
8092 IP = nullptr;
8093 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
8094 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
8095 }
8096 }
8097
8098 // Allocate a new MachineSDNode.
8099 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
8100 createOperands(N, Ops);
8101
8102 if (DoCSE)
8103 CSEMap.InsertNode(N, IP);
8104
8105 InsertNode(N);
8106 return N;
8107}
8108
8109/// getTargetExtractSubreg - A convenience function for creating
8110/// TargetOpcode::EXTRACT_SUBREG nodes.
8111SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8112 SDValue Operand) {
8113 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8114 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
8115 VT, Operand, SRIdxVal);
8116 return SDValue(Subreg, 0);
8117}
8118
8119/// getTargetInsertSubreg - A convenience function for creating
8120/// TargetOpcode::INSERT_SUBREG nodes.
8121SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
8122 SDValue Operand, SDValue Subreg) {
8123 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
8124 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
8125 VT, Operand, Subreg, SRIdxVal);
8126 return SDValue(Result, 0);
8127}
8128
8129/// getNodeIfExists - Get the specified node if it's already available, or
8130/// else return NULL.
8131SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
8132 ArrayRef<SDValue> Ops,
8133 const SDNodeFlags Flags) {
8134 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
8135 FoldingSetNodeID ID;
8136 AddNodeIDNode(ID, Opcode, VTList, Ops);
8137 void *IP = nullptr;
8138 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
8139 E->intersectFlagsWith(Flags);
8140 return E;
8141 }
8142 }
8143 return nullptr;
8144}
8145
8146/// getDbgValue - Creates a SDDbgValue node.
8147///
8148/// SDNode
8149SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
8150 SDNode *N, unsigned R, bool IsIndirect,
8151 const DebugLoc &DL, unsigned O) {
8152 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8153 "Expected inlined-at fields to agree");
8154 return new (DbgInfo->getAlloc())
8155 SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
8156}
8157
8158/// Constant
8159SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
8160 DIExpression *Expr,
8161 const Value *C,
8162 const DebugLoc &DL, unsigned O) {
8163 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8164 "Expected inlined-at fields to agree");
8165 return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
8166}
8167
8168/// FrameIndex
8169SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
8170 DIExpression *Expr, unsigned FI,
8171 bool IsIndirect,
8172 const DebugLoc &DL,
8173 unsigned O) {
8174 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8175 "Expected inlined-at fields to agree");
8176 return new (DbgInfo->getAlloc())
8177 SDDbgValue(Var, Expr, FI, IsIndirect, DL, O, SDDbgValue::FRAMEIX);
8178}
8179
8180/// VReg
8181SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var,
8182 DIExpression *Expr,
8183 unsigned VReg, bool IsIndirect,
8184 const DebugLoc &DL, unsigned O) {
8185 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
8186 "Expected inlined-at fields to agree");
8187 return new (DbgInfo->getAlloc())
8188 SDDbgValue(Var, Expr, VReg, IsIndirect, DL, O, SDDbgValue::VREG);
8189}
8190
8191void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
8192 unsigned OffsetInBits, unsigned SizeInBits,
8193 bool InvalidateDbg) {
8194 SDNode *FromNode = From.getNode();
8195 SDNode *ToNode = To.getNode();
8196 assert(FromNode && ToNode && "Can't modify dbg values");
8197
8198 // PR35338
8199 // TODO: assert(From != To && "Redundant dbg value transfer");
8200 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
8201 if (From == To || FromNode == ToNode)
8202 return;
8203
8204 if (!FromNode->getHasDebugValue())
8205 return;
8206
8207 SmallVector<SDDbgValue *, 2> ClonedDVs;
8208 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
8209 if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
8210 continue;
8211
8212 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
8213
8214 // Just transfer the dbg value attached to From.
8215 if (Dbg->getResNo() != From.getResNo())
8216 continue;
8217
8218 DIVariable *Var = Dbg->getVariable();
8219 auto *Expr = Dbg->getExpression();
8220 // If a fragment is requested, update the expression.
8221 if (SizeInBits) {
8222 // When splitting a larger (e.g., sign-extended) value whose
8223 // lower bits are described with an SDDbgValue, do not attempt
8224 // to transfer the SDDbgValue to the upper bits.
8225 if (auto FI = Expr->getFragmentInfo())
8226 if (OffsetInBits + SizeInBits > FI->SizeInBits)
8227 continue;
8228 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
8229 SizeInBits);
8230 if (!Fragment)
8231 continue;
8232 Expr = *Fragment;
8233 }
8234 // Clone the SDDbgValue and move it to To.
8235 SDDbgValue *Clone =
8236 getDbgValue(Var, Expr, ToNode, To.getResNo(), Dbg->isIndirect(),
8237 Dbg->getDebugLoc(), Dbg->getOrder());
8238 ClonedDVs.push_back(Clone);
8239
8240 if (InvalidateDbg) {
8241 // Invalidate value and indicate the SDDbgValue should not be emitted.
8242 Dbg->setIsInvalidated();
8243 Dbg->setIsEmitted();
8244 }
8245 }
8246
8247 for (SDDbgValue *Dbg : ClonedDVs)
8248 AddDbgValue(Dbg, ToNode, false);
8249}
8250
8251void SelectionDAG::salvageDebugInfo(SDNode &N) {
8252 if (!N.getHasDebugValue())
8253 return;
8254
8255 SmallVector<SDDbgValue *, 2> ClonedDVs;
8256 for (auto DV : GetDbgValues(&N)) {
8257 if (DV->isInvalidated())
8258 continue;
8259 switch (N.getOpcode()) {
8260 default:
8261 break;
8262 case ISD::ADD:
8263 SDValue N0 = N.getOperand(0);
8264 SDValue N1 = N.getOperand(1);
8265 if (!isConstantIntBuildVectorOrConstantInt(N0) &&
8266 isConstantIntBuildVectorOrConstantInt(N1)) {
8267 uint64_t Offset = N.getConstantOperandVal(1);
8268 // Rewrite an ADD constant node into a DIExpression. Since we are
8269 // performing arithmetic to compute the variable's *value* in the
8270 // DIExpression, we need to mark the expression with a
8271 // DW_OP_stack_value.
8272 auto *DIExpr = DV->getExpression();
8273 DIExpr =
8274 DIExpression::prepend(DIExpr, DIExpression::StackValue, Offset);
8275 SDDbgValue *Clone =
8276 getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(),
8277 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder());
8278 ClonedDVs.push_back(Clone);
8279 DV->setIsInvalidated();
8280 DV->setIsEmitted();
8281 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
8282 N0.getNode()->dumprFull(this);
8283 dbgs() << " into " << *DIExpr << '\n');
8284 }
8285 }
8286 }
8287
8288 for (SDDbgValue *Dbg : ClonedDVs)
8289 AddDbgValue(Dbg, Dbg->getSDNode(), false);
8290}
8291
8292/// Creates a SDDbgLabel node.
8293SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
8294 const DebugLoc &DL, unsigned O) {
8295 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
8296 "Expected inlined-at fields to agree");
8297 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
8298}
8299
8300namespace {
8301
8302/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
8303/// pointed to by a use iterator is deleted, increment the use iterator
8304/// so that it doesn't dangle.
8305///
8306class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
8307 SDNode::use_iterator &UI;
8308 SDNode::use_iterator &UE;
8309
8310 void NodeDeleted(SDNode *N, SDNode *E) override {
8311 // Increment the iterator as needed.
8312 while (UI != UE && N == *UI)
8313 ++UI;
8314 }
8315
8316public:
8317 RAUWUpdateListener(SelectionDAG &d,
8318 SDNode::use_iterator &ui,
8319 SDNode::use_iterator &ue)
8320 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
8321};
8322
8323} // end anonymous namespace
8324
8325/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8326/// This can cause recursive merging of nodes in the DAG.
8327///
8328/// This version assumes From has a single result value.
8329///
8330void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
8331 SDNode *From = FromN.getNode();
8332 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
8333 "Cannot replace with this method!");
8334 assert(From != To.getNode() && "Cannot replace uses of with self");
8335
8336 // Preserve Debug Values
8337 transferDbgValues(FromN, To);
8338
8339 // Iterate over all the existing uses of From. New uses will be added
8340 // to the beginning of the use list, which we avoid visiting.
8341 // This specifically avoids visiting uses of From that arise while the
8342 // replacement is happening, because any such uses would be the result
8343 // of CSE: If an existing node looks like From after one of its operands
8344 // is replaced by To, we don't want to replace of all its users with To
8345 // too. See PR3018 for more info.
8346 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8347 RAUWUpdateListener Listener(*this, UI, UE);
8348 while (UI != UE) {
8349 SDNode *User = *UI;
8350
8351 // This node is about to morph, remove its old self from the CSE maps.
8352 RemoveNodeFromCSEMaps(User);
8353
8354 // A user can appear in a use list multiple times, and when this
8355 // happens the uses are usually next to each other in the list.
8356 // To help reduce the number of CSE recomputations, process all
8357 // the uses of this user that we can find this way.
8358 do {
8359 SDUse &Use = UI.getUse();
8360 ++UI;
8361 Use.set(To);
8362 if (To->isDivergent() != From->isDivergent())
8363 updateDivergence(User);
8364 } while (UI != UE && *UI == User);
8365 // Now that we have modified User, add it back to the CSE maps. If it
8366 // already exists there, recursively merge the results together.
8367 AddModifiedNodeToCSEMaps(User);
8368 }
8369
8370 // If we just RAUW'd the root, take note.
8371 if (FromN == getRoot())
8372 setRoot(To);
8373}
8374
8375/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8376/// This can cause recursive merging of nodes in the DAG.
8377///
8378/// This version assumes that for each value of From, there is a
8379/// corresponding value in To in the same position with the same type.
8380///
8381void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
8382#ifndef NDEBUG
8383 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8384 assert((!From->hasAnyUseOfValue(i) ||
8385 From->getValueType(i) == To->getValueType(i)) &&
8386 "Cannot use this version of ReplaceAllUsesWith!");
8387#endif
8388
8389 // Handle the trivial case.
8390 if (From == To)
8391 return;
8392
8393 // Preserve Debug Info. Only do this if there's a use.
8394 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8395 if (From->hasAnyUseOfValue(i)) {
8396 assert((i < To->getNumValues()) && "Invalid To location");
8397 transferDbgValues(SDValue(From, i), SDValue(To, i));
8398 }
8399
8400 // Iterate over just the existing users of From. See the comments in
8401 // the ReplaceAllUsesWith above.
8402 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8403 RAUWUpdateListener Listener(*this, UI, UE);
8404 while (UI != UE) {
8405 SDNode *User = *UI;
8406
8407 // This node is about to morph, remove its old self from the CSE maps.
8408 RemoveNodeFromCSEMaps(User);
8409
8410 // A user can appear in a use list multiple times, and when this
8411 // happens the uses are usually next to each other in the list.
8412 // To help reduce the number of CSE recomputations, process all
8413 // the uses of this user that we can find this way.
8414 do {
8415 SDUse &Use = UI.getUse();
8416 ++UI;
8417 Use.setNode(To);
8418 if (To->isDivergent() != From->isDivergent())
8419 updateDivergence(User);
8420 } while (UI != UE && *UI == User);
8421
8422 // Now that we have modified User, add it back to the CSE maps. If it
8423 // already exists there, recursively merge the results together.
8424 AddModifiedNodeToCSEMaps(User);
8425 }
8426
8427 // If we just RAUW'd the root, take note.
8428 if (From == getRoot().getNode())
8429 setRoot(SDValue(To, getRoot().getResNo()));
8430}
8431
8432/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
8433/// This can cause recursive merging of nodes in the DAG.
8434///
8435/// This version can replace From with any result values. To must match the
8436/// number and types of values returned by From.
8437void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
8438 if (From->getNumValues() == 1) // Handle the simple case efficiently.
8439 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
8440
8441 // Preserve Debug Info.
8442 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
8443 transferDbgValues(SDValue(From, i), To[i]);
8444
8445 // Iterate over just the existing users of From. See the comments in
8446 // the ReplaceAllUsesWith above.
8447 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
8448 RAUWUpdateListener Listener(*this, UI, UE);
8449 while (UI != UE) {
8450 SDNode *User = *UI;
8451
8452 // This node is about to morph, remove its old self from the CSE maps.
8453 RemoveNodeFromCSEMaps(User);
8454
8455 // A user can appear in a use list multiple times, and when this happens the
8456 // uses are usually next to each other in the list. To help reduce the
8457 // number of CSE and divergence recomputations, process all the uses of this
8458 // user that we can find this way.
8459 bool To_IsDivergent = false;
8460 do {
8461 SDUse &Use = UI.getUse();
8462 const SDValue &ToOp = To[Use.getResNo()];
8463 ++UI;
8464 Use.set(ToOp);
8465 To_IsDivergent |= ToOp->isDivergent();
8466 } while (UI != UE && *UI == User);
8467
8468 if (To_IsDivergent != From->isDivergent())
8469 updateDivergence(User);
8470
8471 // Now that we have modified User, add it back to the CSE maps. If it
8472 // already exists there, recursively merge the results together.
8473 AddModifiedNodeToCSEMaps(User);
8474 }
8475
8476 // If we just RAUW'd the root, take note.
8477 if (From == getRoot().getNode())
8478 setRoot(SDValue(To[getRoot().getResNo()]));
8479}
8480
8481/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
8482/// uses of other values produced by From.getNode() alone. The Deleted
8483/// vector is handled the same way as for ReplaceAllUsesWith.
8484void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
8485 // Handle the really simple, really trivial case efficiently.
8486 if (From == To) return;
8487
8488 // Handle the simple, trivial, case efficiently.
8489 if (From.getNode()->getNumValues() == 1) {
8490 ReplaceAllUsesWith(From, To);
8491 return;
8492 }
8493
8494 // Preserve Debug Info.
8495 transferDbgValues(From, To);
8496
8497 // Iterate over just the existing users of From. See the comments in
8498 // the ReplaceAllUsesWith above.
8499 SDNode::use_iterator UI = From.getNode()->use_begin(),
8500 UE = From.getNode()->use_end();
8501 RAUWUpdateListener Listener(*this, UI, UE);
8502 while (UI != UE) {
8503 SDNode *User = *UI;
8504 bool UserRemovedFromCSEMaps = false;
8505
8506 // A user can appear in a use list multiple times, and when this
8507 // happens the uses are usually next to each other in the list.
8508 // To help reduce the number of CSE recomputations, process all
8509 // the uses of this user that we can find this way.
8510 do {
8511 SDUse &Use = UI.getUse();
8512
8513 // Skip uses of different values from the same node.
8514 if (Use.getResNo() != From.getResNo()) {
8515 ++UI;
8516 continue;
8517 }
8518
8519 // If this node hasn't been modified yet, it's still in the CSE maps,
8520 // so remove its old self from the CSE maps.
8521 if (!UserRemovedFromCSEMaps) {
8522 RemoveNodeFromCSEMaps(User);
8523 UserRemovedFromCSEMaps = true;
8524 }
8525
8526 ++UI;
8527 Use.set(To);
8528 if (To->isDivergent() != From->isDivergent())
8529 updateDivergence(User);
8530 } while (UI != UE && *UI == User);
8531 // We are iterating over all uses of the From node, so if a use
8532 // doesn't use the specific value, no changes are made.
8533 if (!UserRemovedFromCSEMaps)
8534 continue;
8535
8536 // Now that we have modified User, add it back to the CSE maps. If it
8537 // already exists there, recursively merge the results together.
8538 AddModifiedNodeToCSEMaps(User);
8539 }
8540
8541 // If we just RAUW'd the root, take note.
8542 if (From == getRoot())
8543 setRoot(To);
8544}
8545
8546namespace {
8547
8548 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
8549 /// to record information about a use.
8550 struct UseMemo {
8551 SDNode *User;
8552 unsigned Index;
8553 SDUse *Use;
8554 };
8555
8556 /// operator< - Sort Memos by User.
8557 bool operator<(const UseMemo &L, const UseMemo &R) {
8558 return (intptr_t)L.User < (intptr_t)R.User;
8559 }
8560
8561} // end anonymous namespace
8562
8563void SelectionDAG::updateDivergence(SDNode * N)
8564{
8565 if (TLI->isSDNodeAlwaysUniform(N))
8566 return;
8567 bool IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
8568 for (auto &Op : N->ops()) {
8569 if (Op.Val.getValueType() != MVT::Other)
8570 IsDivergent |= Op.getNode()->isDivergent();
8571 }
8572 if (N->SDNodeBits.IsDivergent != IsDivergent) {
8573 N->SDNodeBits.IsDivergent = IsDivergent;
8574 for (auto U : N->uses()) {
8575 updateDivergence(U);
8576 }
8577 }
8578}
8579
8580void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
8581 DenseMap<SDNode *, unsigned> Degree;
8582 Order.reserve(AllNodes.size());
8583 for (auto &N : allnodes()) {
8584 unsigned NOps = N.getNumOperands();
8585 Degree[&N] = NOps;
8586 if (0 == NOps)
8587 Order.push_back(&N);
8588 }
8589 for (size_t I = 0; I != Order.size(); ++I) {
8590 SDNode *N = Order[I];
8591 for (auto U : N->uses()) {
8592 unsigned &UnsortedOps = Degree[U];
8593 if (0 == --UnsortedOps)
8594 Order.push_back(U);
8595 }
8596 }
8597}
8598
8599#ifndef NDEBUG
8600void SelectionDAG::VerifyDAGDiverence() {
8601 std::vector<SDNode *> TopoOrder;
8602 CreateTopologicalOrder(TopoOrder);
8603 const TargetLowering &TLI = getTargetLoweringInfo();
8604 DenseMap<const SDNode *, bool> DivergenceMap;
8605 for (auto &N : allnodes()) {
8606 DivergenceMap[&N] = false;
8607 }
8608 for (auto N : TopoOrder) {
8609 bool IsDivergent = DivergenceMap[N];
8610 bool IsSDNodeDivergent = TLI.isSDNodeSourceOfDivergence(N, FLI, DA);
8611 for (auto &Op : N->ops()) {
8612 if (Op.Val.getValueType() != MVT::Other)
8613 IsSDNodeDivergent |= DivergenceMap[Op.getNode()];
8614 }
8615 if (!IsDivergent && IsSDNodeDivergent && !TLI.isSDNodeAlwaysUniform(N)) {
8616 DivergenceMap[N] = true;
8617 }
8618 }
8619 for (auto &N : allnodes()) {
8620 (void)N;
8621 assert(DivergenceMap[&N] == N.isDivergent() &&
8622 "Divergence bit inconsistency detected\n");
8623 }
8624}
8625#endif
8626
8627/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
8628/// uses of other values produced by From.getNode() alone. The same value
8629/// may appear in both the From and To list. The Deleted vector is
8630/// handled the same way as for ReplaceAllUsesWith.
8631void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
8632 const SDValue *To,
8633 unsigned Num){
8634 // Handle the simple, trivial case efficiently.
8635 if (Num == 1)
8636 return ReplaceAllUsesOfValueWith(*From, *To);
8637
8638 transferDbgValues(*From, *To);
8639
8640 // Read up all the uses and make records of them. This helps
8641 // processing new uses that are introduced during the
8642 // replacement process.
8643 SmallVector<UseMemo, 4> Uses;
8644 for (unsigned i = 0; i != Num; ++i) {
8645 unsigned FromResNo = From[i].getResNo();
8646 SDNode *FromNode = From[i].getNode();
8647 for (SDNode::use_iterator UI = FromNode->use_begin(),
8648 E = FromNode->use_end(); UI != E; ++UI) {
8649 SDUse &Use = UI.getUse();
8650 if (Use.getResNo() == FromResNo) {
8651 UseMemo Memo = { *UI, i, &Use };
8652 Uses.push_back(Memo);
8653 }
8654 }
8655 }
8656
8657 // Sort the uses, so that all the uses from a given User are together.
8658 llvm::sort(Uses);
8659
8660 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
8661 UseIndex != UseIndexEnd; ) {
8662 // We know that this user uses some value of From. If it is the right
8663 // value, update it.
8664 SDNode *User = Uses[UseIndex].User;
8665
8666 // This node is about to morph, remove its old self from the CSE maps.
8667 RemoveNodeFromCSEMaps(User);
8668
8669 // The Uses array is sorted, so all the uses for a given User
8670 // are next to each other in the list.
8671 // To help reduce the number of CSE recomputations, process all
8672 // the uses of this user that we can find this way.
8673 do {
8674 unsigned i = Uses[UseIndex].Index;
8675 SDUse &Use = *Uses[UseIndex].Use;
8676 ++UseIndex;
8677
8678 Use.set(To[i]);
8679 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
8680
8681 // Now that we have modified User, add it back to the CSE maps. If it
8682 // already exists there, recursively merge the results together.
8683 AddModifiedNodeToCSEMaps(User);
8684 }
8685}
8686
8687/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
8688/// based on their topological order. It returns the maximum id and a vector
8689/// of the SDNodes* in assigned order by reference.
8690unsigned SelectionDAG::AssignTopologicalOrder() {
8691 unsigned DAGSize = 0;
8692
8693 // SortedPos tracks the progress of the algorithm. Nodes before it are
8694 // sorted, nodes after it are unsorted. When the algorithm completes
8695 // it is at the end of the list.
8696 allnodes_iterator SortedPos = allnodes_begin();
8697
8698 // Visit all the nodes. Move nodes with no operands to the front of
8699 // the list immediately. Annotate nodes that do have operands with their
8700 // operand count. Before we do this, the Node Id fields of the nodes
8701 // may contain arbitrary values. After, the Node Id fields for nodes
8702 // before SortedPos will contain the topological sort index, and the
8703 // Node Id fields for nodes At SortedPos and after will contain the
8704 // count of outstanding operands.
8705 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
8706 SDNode *N = &*I++;
8707 checkForCycles(N, this);
8708 unsigned Degree = N->getNumOperands();
8709 if (Degree == 0) {
8710 // A node with no uses, add it to the result array immediately.
8711 N->setNodeId(DAGSize++);
8712 allnodes_iterator Q(N);
8713 if (Q != SortedPos)
8714 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
8715 assert(SortedPos != AllNodes.end() && "Overran node list");
8716 ++SortedPos;
8717 } else {
8718 // Temporarily use the Node Id as scratch space for the degree count.
8719 N->setNodeId(Degree);
8720 }
8721 }
8722
8723 // Visit all the nodes. As we iterate, move nodes into sorted order,
8724 // such that by the time the end is reached all nodes will be sorted.
8725 for (SDNode &Node : allnodes()) {
8726 SDNode *N = &Node;
8727 checkForCycles(N, this);
8728 // N is in sorted position, so all its uses have one less operand
8729 // that needs to be sorted.
8730 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
8731 UI != UE; ++UI) {
8732 SDNode *P = *UI;
8733 unsigned Degree = P->getNodeId();
8734 assert(Degree != 0 && "Invalid node degree");
8735 --Degree;
8736 if (Degree == 0) {
8737 // All of P's operands are sorted, so P may sorted now.
8738 P->setNodeId(DAGSize++);
8739 if (P->getIterator() != SortedPos)
8740 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
8741 assert(SortedPos != AllNodes.end() && "Overran node list");
8742 ++SortedPos;
8743 } else {
8744 // Update P's outstanding operand count.
8745 P->setNodeId(Degree);
8746 }
8747 }
8748 if (Node.getIterator() == SortedPos) {
8749#ifndef NDEBUG
8750 allnodes_iterator I(N);
8751 SDNode *S = &*++I;
8752 dbgs() << "Overran sorted position:\n";
8753 S->dumprFull(this); dbgs() << "\n";
8754 dbgs() << "Checking if this is due to cycles\n";
8755 checkForCycles(this, true);
8756#endif
8757 llvm_unreachable(nullptr);
8758 }
8759 }
8760
8761 assert(SortedPos == AllNodes.end() &&
8762 "Topological sort incomplete!");
8763 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
8764 "First node in topological sort is not the entry token!");
8765 assert(AllNodes.front().getNodeId() == 0 &&
8766 "First node in topological sort has non-zero id!");
8767 assert(AllNodes.front().getNumOperands() == 0 &&
8768 "First node in topological sort has operands!");
8769 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
8770 "Last node in topologic sort has unexpected id!");
8771 assert(AllNodes.back().use_empty() &&
8772 "Last node in topologic sort has users!");
8773 assert(DAGSize == allnodes_size() && "Node count mismatch!");
8774 return DAGSize;
8775}
8776
8777/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
8778/// value is produced by SD.
8779void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
8780 if (SD) {
8781 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
8782 SD->setHasDebugValue(true);
8783 }
8784 DbgInfo->add(DB, SD, isParameter);
8785}
8786
8787void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) {
8788 DbgInfo->add(DB);
8789}
8790
8791SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
8792 SDValue NewMemOp) {
8793 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
8794 // The new memory operation must have the same position as the old load in
8795 // terms of memory dependency. Create a TokenFactor for the old load and new
8796 // memory operation and update uses of the old load's output chain to use that
8797 // TokenFactor.
8798 SDValue OldChain = SDValue(OldLoad, 1);
8799 SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
8800 if (!OldLoad->hasAnyUseOfValue(1))
8801 return NewChain;
8802
8803 SDValue TokenFactor =
8804 getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
8805 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
8806 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
8807 return TokenFactor;
8808}
8809
8810SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
8811 Function **OutFunction) {
8812 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
8813
8814 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
8815 auto *Module = MF->getFunction().getParent();
8816 auto *Function = Module->getFunction(Symbol);
8817
8818 if (OutFunction != nullptr)
8819 *OutFunction = Function;
8820
8821 if (Function != nullptr) {
8822 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
8823 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
8824 }
8825
8826 std::string ErrorStr;
8827 raw_string_ostream ErrorFormatter(ErrorStr);
8828
8829 ErrorFormatter << "Undefined external symbol ";
8830 ErrorFormatter << '"' << Symbol << '"';
8831 ErrorFormatter.flush();
8832
8833 report_fatal_error(ErrorStr);
8834}
8835
8836//===----------------------------------------------------------------------===//
8837// SDNode Class
8838//===----------------------------------------------------------------------===//
8839
8840bool llvm::isNullConstant(SDValue V) {
8841 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8842 return Const != nullptr && Const->isNullValue();
8843}
8844
8845bool llvm::isNullFPConstant(SDValue V) {
8846 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
8847 return Const != nullptr && Const->isZero() && !Const->isNegative();
8848}
8849
8850bool llvm::isAllOnesConstant(SDValue V) {
8851 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8852 return Const != nullptr && Const->isAllOnesValue();
8853}
8854
8855bool llvm::isOneConstant(SDValue V) {
8856 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
8857 return Const != nullptr && Const->isOne();
8858}
8859
8860SDValue llvm::peekThroughBitcasts(SDValue V) {
8861 while (V.getOpcode() == ISD::BITCAST)
8862 V = V.getOperand(0);
8863 return V;
8864}
8865
8866SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
8867 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
8868 V = V.getOperand(0);
8869 return V;
8870}
8871
8872SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
8873 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
8874 V = V.getOperand(0);
8875 return V;
8876}
8877
8878bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
8879 if (V.getOpcode() != ISD::XOR)
8880 return false;
8881 V = peekThroughBitcasts(V.getOperand(1));
8882 unsigned NumBits = V.getScalarValueSizeInBits();
8883 ConstantSDNode *C =
8884 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
8885 return C && (C->getAPIntValue().countTrailingOnes() >= NumBits);
8886}
8887
8888ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
8889 bool AllowTruncation) {
8890 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
8891 return CN;
8892
8893 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8894 BitVector UndefElements;
8895 ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
8896
8897 // BuildVectors can truncate their operands. Ignore that case here unless
8898 // AllowTruncation is set.
8899 if (CN && (UndefElements.none() || AllowUndefs)) {
8900 EVT CVT = CN->getValueType(0);
8901 EVT NSVT = N.getValueType().getScalarType();
8902 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
8903 if (AllowTruncation || (CVT == NSVT))
8904 return CN;
8905 }
8906 }
8907
8908 return nullptr;
8909}
8910
8911ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
8912 bool AllowUndefs,
8913 bool AllowTruncation) {
8914 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
8915 return CN;
8916
8917 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8918 BitVector UndefElements;
8919 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
8920
8921 // BuildVectors can truncate their operands. Ignore that case here unless
8922 // AllowTruncation is set.
8923 if (CN && (UndefElements.none() || AllowUndefs)) {
8924 EVT CVT = CN->getValueType(0);
8925 EVT NSVT = N.getValueType().getScalarType();
8926 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
8927 if (AllowTruncation || (CVT == NSVT))
8928 return CN;
8929 }
8930 }
8931
8932 return nullptr;
8933}
8934
8935ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
8936 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
8937 return CN;
8938
8939 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8940 BitVector UndefElements;
8941 ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
8942 if (CN && (UndefElements.none() || AllowUndefs))
8943 return CN;
8944 }
8945
8946 return nullptr;
8947}
8948
8949ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
8950 const APInt &DemandedElts,
8951 bool AllowUndefs) {
8952 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
8953 return CN;
8954
8955 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
8956 BitVector UndefElements;
8957 ConstantFPSDNode *CN =
8958 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
8959 if (CN && (UndefElements.none() || AllowUndefs))
8960 return CN;
8961 }
8962
8963 return nullptr;
8964}
8965
8966bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
8967 // TODO: may want to use peekThroughBitcast() here.
8968 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
8969 return C && C->isNullValue();
8970}
8971
8972bool llvm::isOneOrOneSplat(SDValue N) {
8973 // TODO: may want to use peekThroughBitcast() here.
8974 unsigned BitWidth = N.getScalarValueSizeInBits();
8975 ConstantSDNode *C = isConstOrConstSplat(N);
8976 return C && C->isOne() && C->getValueSizeInBits(0) == BitWidth;
8977}
8978
8979bool llvm::isAllOnesOrAllOnesSplat(SDValue N) {
8980 N = peekThroughBitcasts(N);
8981 unsigned BitWidth = N.getScalarValueSizeInBits();
8982 ConstantSDNode *C = isConstOrConstSplat(N);
8983 return C && C->isAllOnesValue() && C->getValueSizeInBits(0) == BitWidth;
8984}
8985
8986HandleSDNode::~HandleSDNode() {
8987 DropOperands();
8988}
8989
8990GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
8991 const DebugLoc &DL,
8992 const GlobalValue *GA, EVT VT,
8993 int64_t o, unsigned char TF)
8994 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
8995 TheGlobal = GA;
8996}
8997
8998AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
8999 EVT VT, unsigned SrcAS,
9000 unsigned DestAS)
9001 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
9002 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
9003
9004MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
9005 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
9006 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
9007 MemSDNodeBits.IsVolatile = MMO->isVolatile();
9008 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
9009 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
9010 MemSDNodeBits.IsInvariant = MMO->isInvariant();
9011
9012 // We check here that the size of the memory operand fits within the size of
9013 // the MMO. This is because the MMO might indicate only a possible address
9014 // range instead of specifying the affected memory addresses precisely.
9015 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
9016}
9017
9018/// Profile - Gather unique data for the node.
9019///
9020void SDNode::Profile(FoldingSetNodeID &ID) const {
9021 AddNodeIDNode(ID, this);
9022}
9023
9024namespace {
9025
9026 struct EVTArray {
9027 std::vector<EVT> VTs;
9028
9029 EVTArray() {
9030 VTs.reserve(MVT::LAST_VALUETYPE);
9031 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
9032 VTs.push_back(MVT((MVT::SimpleValueType)i));
9033 }
9034 };
9035
9036} // end anonymous namespace
9037
9038static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
9039static ManagedStatic<EVTArray> SimpleVTArray;
9040static ManagedStatic<sys::SmartMutex<true>> VTMutex;
9041
9042/// getValueTypeList - Return a pointer to the specified value type.
9043///
9044const EVT *SDNode::getValueTypeList(EVT VT) {
9045 if (VT.isExtended()) {
9046 sys::SmartScopedLock<true> Lock(*VTMutex);
9047 return &(*EVTs->insert(VT).first);
9048 } else {
9049 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
9050 "Value type out of range!");
9051 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
9052 }
9053}
9054
9055/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
9056/// indicated value. This method ignores uses of other values defined by this
9057/// operation.
9058bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
9059 assert(Value < getNumValues() && "Bad value!");
9060
9061 // TODO: Only iterate over uses of a given value of the node
9062 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
9063 if (UI.getUse().getResNo() == Value) {
9064 if (NUses == 0)
9065 return false;
9066 --NUses;
9067 }
9068 }
9069
9070 // Found exactly the right number of uses?
9071 return NUses == 0;
9072}
9073
9074/// hasAnyUseOfValue - Return true if there are any use of the indicated
9075/// value. This method ignores uses of other values defined by this operation.
9076bool SDNode::hasAnyUseOfValue(unsigned Value) const {
9077 assert(Value < getNumValues() && "Bad value!");
9078
9079 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
9080 if (UI.getUse().getResNo() == Value)
9081 return true;
9082
9083 return false;
9084}
9085
9086/// isOnlyUserOf - Return true if this node is the only use of N.
9087bool SDNode::isOnlyUserOf(const SDNode *N) const {
9088 bool Seen = false;
9089 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9090 SDNode *User = *I;
9091 if (User == this)
9092 Seen = true;
9093 else
9094 return false;
9095 }
9096
9097 return Seen;
9098}
9099
9100/// Return true if the only users of N are contained in Nodes.
9101bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
9102 bool Seen = false;
9103 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
9104 SDNode *User = *I;
9105 if (llvm::any_of(Nodes,
9106 [&User](const SDNode *Node) { return User == Node; }))
9107 Seen = true;
9108 else
9109 return false;
9110 }
9111
9112 return Seen;
9113}
9114
9115/// isOperand - Return true if this node is an operand of N.
9116bool SDValue::isOperandOf(const SDNode *N) const {
9117 return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; });
9118}
9119
9120bool SDNode::isOperandOf(const SDNode *N) const {
9121 return any_of(N->op_values(),
9122 [this](SDValue Op) { return this == Op.getNode(); });
9123}
9124
9125/// reachesChainWithoutSideEffects - Return true if this operand (which must
9126/// be a chain) reaches the specified operand without crossing any
9127/// side-effecting instructions on any chain path. In practice, this looks
9128/// through token factors and non-volatile loads. In order to remain efficient,
9129/// this only looks a couple of nodes in, it does not do an exhaustive search.
9130///
9131/// Note that we only need to examine chains when we're searching for
9132/// side-effects; SelectionDAG requires that all side-effects are represented
9133/// by chains, even if another operand would force a specific ordering. This
9134/// constraint is necessary to allow transformations like splitting loads.
9135bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
9136 unsigned Depth) const {
9137 if (*this == Dest) return true;
9138
9139 // Don't search too deeply, we just want to be able to see through
9140 // TokenFactor's etc.
9141 if (Depth == 0) return false;
9142
9143 // If this is a token factor, all inputs to the TF happen in parallel.
9144 if (getOpcode() == ISD::TokenFactor) {
9145 // First, try a shallow search.
9146 if (is_contained((*this)->ops(), Dest)) {
9147 // We found the chain we want as an operand of this TokenFactor.
9148 // Essentially, we reach the chain without side-effects if we could
9149 // serialize the TokenFactor into a simple chain of operations with
9150 // Dest as the last operation. This is automatically true if the
9151 // chain has one use: there are no other ordering constraints.
9152 // If the chain has more than one use, we give up: some other
9153 // use of Dest might force a side-effect between Dest and the current
9154 // node.
9155 if (Dest.hasOneUse())
9156 return true;
9157 }
9158 // Next, try a deep search: check whether every operand of the TokenFactor
9159 // reaches Dest.
9160 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
9161 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
9162 });
9163 }
9164
9165 // Loads don't have side effects, look through them.
9166 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
9167 if (!Ld->isVolatile())
9168 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
9169 }
9170 return false;
9171}
9172
9173bool SDNode::hasPredecessor(const SDNode *N) const {
9174 SmallPtrSet<const SDNode *, 32> Visited;
9175 SmallVector<const SDNode *, 16> Worklist;
9176 Worklist.push_back(this);
9177 return hasPredecessorHelper(N, Visited, Worklist);
9178}
9179
9180void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
9181 this->Flags.intersectWith(Flags);
9182}
9183
9184SDValue
9185SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
9186 ArrayRef<ISD::NodeType> CandidateBinOps) {
9187 // The pattern must end in an extract from index 0.
9188 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9189 !isNullConstant(Extract->getOperand(1)))
9190 return SDValue();
9191
9192 SDValue Op = Extract->getOperand(0);
9193 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
9194
9195 // Match against one of the candidate binary ops.
9196 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
9197 return Op.getOpcode() == unsigned(BinOp);
9198 }))
9199 return SDValue();
9200
9201 // At each stage, we're looking for something that looks like:
9202 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
9203 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
9204 // i32 undef, i32 undef, i32 undef, i32 undef>
9205 // %a = binop <8 x i32> %op, %s
9206 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
9207 // we expect something like:
9208 // <4,5,6,7,u,u,u,u>
9209 // <2,3,u,u,u,u,u,u>
9210 // <1,u,u,u,u,u,u,u>
9211 unsigned CandidateBinOp = Op.getOpcode();
9212 for (unsigned i = 0; i < Stages; ++i) {
9213 if (Op.getOpcode() != CandidateBinOp)
9214 return SDValue();
9215
9216 SDValue Op0 = Op.getOperand(0);
9217 SDValue Op1 = Op.getOperand(1);
9218
9219 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
9220 if (Shuffle) {
9221 Op = Op1;
9222 } else {
9223 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
9224 Op = Op0;
9225 }
9226
9227 // The first operand of the shuffle should be the same as the other operand
9228 // of the binop.
9229 if (!Shuffle || Shuffle->getOperand(0) != Op)
9230 return SDValue();
9231
9232 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
9233 for (int Index = 0, MaskEnd = 1 << i; Index < MaskEnd; ++Index)
9234 if (Shuffle->getMaskElt(Index) != MaskEnd + Index)
9235 return SDValue();
9236 }
9237
9238 BinOp = (ISD::NodeType)CandidateBinOp;
9239 return Op;
9240}
9241
9242SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
9243 assert(N->getNumValues() == 1 &&
9244 "Can't unroll a vector with multiple results!");
9245
9246 EVT VT = N->getValueType(0);
9247 unsigned NE = VT.getVectorNumElements();
9248 EVT EltVT = VT.getVectorElementType();
9249 SDLoc dl(N);
9250
9251 SmallVector<SDValue, 8> Scalars;
9252 SmallVector<SDValue, 4> Operands(N->getNumOperands());
9253
9254 // If ResNE is 0, fully unroll the vector op.
9255 if (ResNE == 0)
9256 ResNE = NE;
9257 else if (NE > ResNE)
9258 NE = ResNE;
9259
9260 unsigned i;
9261 for (i= 0; i != NE; ++i) {
9262 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
9263 SDValue Operand = N->getOperand(j);
9264 EVT OperandVT = Operand.getValueType();
9265 if (OperandVT.isVector()) {
9266 // A vector operand; extract a single element.
9267 EVT OperandEltVT = OperandVT.getVectorElementType();
9268 Operands[j] =
9269 getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
9270 getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
9271 } else {
9272 // A scalar operand; just use it as is.
9273 Operands[j] = Operand;
9274 }
9275 }
9276
9277 switch (N->getOpcode()) {
9278 default: {
9279 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
9280 N->getFlags()));
9281 break;
9282 }
9283 case ISD::VSELECT:
9284 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
9285 break;
9286 case ISD::SHL:
9287 case ISD::SRA:
9288 case ISD::SRL:
9289 case ISD::ROTL:
9290 case ISD::ROTR:
9291 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
9292 getShiftAmountOperand(Operands[0].getValueType(),
9293 Operands[1])));
9294 break;
9295 case ISD::SIGN_EXTEND_INREG:
9296 case ISD::FP_ROUND_INREG: {
9297 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
9298 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
9299 Operands[0],
9300 getValueType(ExtVT)));
9301 }
9302 }
9303 }
9304
9305 for (; i < ResNE; ++i)
9306 Scalars.push_back(getUNDEF(EltVT));
9307
9308 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
9309 return getBuildVector(VecVT, dl, Scalars);
9310}
9311
9312std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
9313 SDNode *N, unsigned ResNE) {
9314 unsigned Opcode = N->getOpcode();
9315 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
9316 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
9317 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
9318 "Expected an overflow opcode");
9319
9320 EVT ResVT = N->getValueType(0);
9321 EVT OvVT = N->getValueType(1);
9322 EVT ResEltVT = ResVT.getVectorElementType();
9323 EVT OvEltVT = OvVT.getVectorElementType();
9324 SDLoc dl(N);
9325
9326 // If ResNE is 0, fully unroll the vector op.
9327 unsigned NE = ResVT.getVectorNumElements();
9328 if (ResNE == 0)
9329 ResNE = NE;
9330 else if (NE > ResNE)
9331 NE = ResNE;
9332
9333 SmallVector<SDValue, 8> LHSScalars;
9334 SmallVector<SDValue, 8> RHSScalars;
9335 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
9336 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
9337
9338 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
9339 SDVTList VTs = getVTList(ResEltVT, SVT);
9340 SmallVector<SDValue, 8> ResScalars;
9341 SmallVector<SDValue, 8> OvScalars;
9342 for (unsigned i = 0; i < NE; ++i) {
9343 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
9344 SDValue Ov =
9345 getSelect(dl, OvEltVT, Res.getValue(1),
9346 getBoolConstant(true, dl, OvEltVT, ResVT),
9347 getConstant(0, dl, OvEltVT));
9348
9349 ResScalars.push_back(Res);
9350 OvScalars.push_back(Ov);
9351 }
9352
9353 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
9354 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
9355
9356 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
9357 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
9358 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
9359 getBuildVector(NewOvVT, dl, OvScalars));
9360}
9361
9362bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
9363 LoadSDNode *Base,
9364 unsigned Bytes,
9365 int Dist) const {
9366 if (LD->isVolatile() || Base->isVolatile())
9367 return false;
9368 if (LD->isIndexed() || Base->isIndexed())
9369 return false;
9370 if (LD->getChain() != Base->getChain())
9371 return false;
9372 EVT VT = LD->getValueType(0);
9373 if (VT.getSizeInBits() / 8 != Bytes)
9374 return false;
9375
9376 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
9377 auto LocDecomp = BaseIndexOffset::match(LD, *this);
9378
9379 int64_t Offset = 0;
9380 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
9381 return (Dist * Bytes == Offset);
9382 return false;
9383}
9384
9385/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
9386/// it cannot be inferred.
9387unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
9388 // If this is a GlobalAddress + cst, return the alignment.
9389 const GlobalValue *GV;
9390 int64_t GVOffset = 0;
9391 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
9392 unsigned IdxWidth = getDataLayout().getIndexTypeSizeInBits(GV->getType());
9393 KnownBits Known(IdxWidth);
9394 llvm::computeKnownBits(GV, Known, getDataLayout());
9395 unsigned AlignBits = Known.countMinTrailingZeros();
9396 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
9397 if (Align)
9398 return MinAlign(Align, GVOffset);
9399 }
9400
9401 // If this is a direct reference to a stack slot, use information about the
9402 // stack slot's alignment.
9403 int FrameIdx = INT_MIN;
9404 int64_t FrameOffset = 0;
9405 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
9406 FrameIdx = FI->getIndex();
9407 } else if (isBaseWithConstantOffset(Ptr) &&
9408 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
9409 // Handle FI+Cst
9410 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
9411 FrameOffset = Ptr.getConstantOperandVal(1);
9412 }
9413
9414 if (FrameIdx != INT_MIN) {
9415 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
9416 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
9417 FrameOffset);
9418 return FIInfoAlign;
9419 }
9420
9421 return 0;
9422}
9423
9424/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
9425/// which is split (or expanded) into two not necessarily identical pieces.
9426std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
9427 // Currently all types are split in half.
9428 EVT LoVT, HiVT;
9429 if (!VT.isVector())
9430 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
9431 else
9432 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
9433
9434 return std::make_pair(LoVT, HiVT);
9435}
9436
9437/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
9438/// low/high part.
9439std::pair<SDValue, SDValue>
9440SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
9441 const EVT &HiVT) {
9442 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
9443 N.getValueType().getVectorNumElements() &&
9444 "More vector elements requested than available!");
9445 SDValue Lo, Hi;
9446 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
9447 getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
9448 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
9449 getConstant(LoVT.getVectorNumElements(), DL,
9450 TLI->getVectorIdxTy(getDataLayout())));
9451 return std::make_pair(Lo, Hi);
9452}
9453
9454/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
9455SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
9456 EVT VT = N.getValueType();
9457 EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
9458 NextPowerOf2(VT.getVectorNumElements()));
9459 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
9460 getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
9461}
9462
9463void SelectionDAG::ExtractVectorElements(SDValue Op,
9464 SmallVectorImpl<SDValue> &Args,
9465 unsigned Start, unsigned Count) {
9466 EVT VT = Op.getValueType();
9467 if (Count == 0)
9468 Count = VT.getVectorNumElements();
9469
9470 EVT EltVT = VT.getVectorElementType();
9471 EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
9472 SDLoc SL(Op);
9473 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
9474 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
9475 Op, getConstant(i, SL, IdxTy)));
9476 }
9477}
9478
9479// getAddressSpace - Return the address space this GlobalAddress belongs to.
9480unsigned GlobalAddressSDNode::getAddressSpace() const {
9481 return getGlobal()->getType()->getAddressSpace();
9482}
9483
9484Type *ConstantPoolSDNode::getType() const {
9485 if (isMachineConstantPoolEntry())
9486 return Val.MachineCPVal->getType();
9487 return Val.ConstVal->getType();
9488}
9489
9490bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
9491 unsigned &SplatBitSize,
9492 bool &HasAnyUndefs,
9493 unsigned MinSplatBits,
9494 bool IsBigEndian) const {
9495 EVT VT = getValueType(0);
9496 assert(VT.isVector() && "Expected a vector type");
9497 unsigned VecWidth = VT.getSizeInBits();
9498 if (MinSplatBits > VecWidth)
9499 return false;
9500
9501 // FIXME: The widths are based on this node's type, but build vectors can
9502 // truncate their operands.
9503 SplatValue = APInt(VecWidth, 0);
9504 SplatUndef = APInt(VecWidth, 0);
9505
9506 // Get the bits. Bits with undefined values (when the corresponding element
9507 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
9508 // in SplatValue. If any of the values are not constant, give up and return
9509 // false.
9510 unsigned int NumOps = getNumOperands();
9511 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
9512 unsigned EltWidth = VT.getScalarSizeInBits();
9513
9514 for (unsigned j = 0; j < NumOps; ++j) {
9515 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
9516 SDValue OpVal = getOperand(i);
9517 unsigned BitPos = j * EltWidth;
9518
9519 if (OpVal.isUndef())
9520 SplatUndef.setBits(BitPos, BitPos + EltWidth);
9521 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
9522 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
9523 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
9524 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
9525 else
9526 return false;
9527 }
9528
9529 // The build_vector is all constants or undefs. Find the smallest element
9530 // size that splats the vector.
9531 HasAnyUndefs = (SplatUndef != 0);
9532
9533 // FIXME: This does not work for vectors with elements less than 8 bits.
9534 while (VecWidth > 8) {
9535 unsigned HalfSize = VecWidth / 2;
9536 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
9537 APInt LowValue = SplatValue.trunc(HalfSize);
9538 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
9539 APInt LowUndef = SplatUndef.trunc(HalfSize);
9540
9541 // If the two halves do not match (ignoring undef bits), stop here.
9542 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
9543 MinSplatBits > HalfSize)
9544 break;
9545
9546 SplatValue = HighValue | LowValue;
9547 SplatUndef = HighUndef & LowUndef;
9548
9549 VecWidth = HalfSize;
9550 }
9551
9552 SplatBitSize = VecWidth;
9553 return true;
9554}
9555
9556SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
9557 BitVector *UndefElements) const {
9558 if (UndefElements) {
9559 UndefElements->clear();
9560 UndefElements->resize(getNumOperands());
9561 }
9562 assert(getNumOperands() == DemandedElts.getBitWidth() &&
9563 "Unexpected vector size");
9564 if (!DemandedElts)
9565 return SDValue();
9566 SDValue Splatted;
9567 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
9568 if (!DemandedElts[i])
9569 continue;
9570 SDValue Op = getOperand(i);
9571 if (Op.isUndef()) {
9572 if (UndefElements)
9573 (*UndefElements)[i] = true;
9574 } else if (!Splatted) {
9575 Splatted = Op;
9576 } else if (Splatted != Op) {
9577 return SDValue();
9578 }
9579 }
9580
9581 if (!Splatted) {
9582 unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros();
9583 assert(getOperand(FirstDemandedIdx).isUndef() &&
9584 "Can only have a splat without a constant for all undefs.");
9585 return getOperand(FirstDemandedIdx);
9586 }
9587
9588 return Splatted;
9589}
9590
9591SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
9592 APInt DemandedElts = APInt::getAllOnesValue(getNumOperands());
9593 return getSplatValue(DemandedElts, UndefElements);
9594}
9595
9596ConstantSDNode *
9597BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
9598 BitVector *UndefElements) const {
9599 return dyn_cast_or_null<ConstantSDNode>(
9600 getSplatValue(DemandedElts, UndefElements));
9601}
9602
9603ConstantSDNode *
9604BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
9605 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
9606}
9607
9608ConstantFPSDNode *
9609BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
9610 BitVector *UndefElements) const {
9611 return dyn_cast_or_null<ConstantFPSDNode>(
9612 getSplatValue(DemandedElts, UndefElements));
9613}
9614
9615ConstantFPSDNode *
9616BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
9617 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
9618}
9619
9620int32_t
9621BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
9622 uint32_t BitWidth) const {
9623 if (ConstantFPSDNode *CN =
9624 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
9625 bool IsExact;
9626 APSInt IntVal(BitWidth);
9627 const APFloat &APF = CN->getValueAPF();
9628 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
9629 APFloat::opOK ||
9630 !IsExact)
9631 return -1;
9632
9633 return IntVal.exactLogBase2();
9634 }
9635 return -1;
9636}
9637
9638bool BuildVectorSDNode::isConstant() const {
9639 for (const SDValue &Op : op_values()) {
9640 unsigned Opc = Op.getOpcode();
9641 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
9642 return false;
9643 }
9644 return true;
9645}
9646
9647bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
9648 // Find the first non-undef value in the shuffle mask.
9649 unsigned i, e;
9650 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
9651 /* search */;
9652
9653 // If all elements are undefined, this shuffle can be considered a splat
9654 // (although it should eventually get simplified away completely).
9655 if (i == e)
9656 return true;
9657
9658 // Make sure all remaining elements are either undef or the same as the first
9659 // non-undef value.
9660 for (int Idx = Mask[i]; i != e; ++i)
9661 if (Mask[i] >= 0 && Mask[i] != Idx)
9662 return false;
9663 return true;
9664}
9665
9666// Returns the SDNode if it is a constant integer BuildVector
9667// or constant integer.
9668SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
9669 if (isa<ConstantSDNode>(N))
9670 return N.getNode();
9671 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
9672 return N.getNode();
9673 // Treat a GlobalAddress supporting constant offset folding as a
9674 // constant integer.
9675 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
9676 if (GA->getOpcode() == ISD::GlobalAddress &&
9677 TLI->isOffsetFoldingLegal(GA))
9678 return GA;
9679 return nullptr;
9680}
9681
9682SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
9683 if (isa<ConstantFPSDNode>(N))
9684 return N.getNode();
9685
9686 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
9687 return N.getNode();
9688
9689 return nullptr;
9690}
9691
9692void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
9693 assert(!Node->OperandList && "Node already has operands");
9694 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
9695 "too many operands to fit into SDNode");
9696 SDUse *Ops = OperandRecycler.allocate(
9697 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
9698
9699 bool IsDivergent = false;
9700 for (unsigned I = 0; I != Vals.size(); ++I) {
9701 Ops[I].setUser(Node);
9702 Ops[I].setInitial(Vals[I]);
9703 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
9704 IsDivergent = IsDivergent || Ops[I].getNode()->isDivergent();
9705 }
9706 Node->NumOperands = Vals.size();
9707 Node->OperandList = Ops;
9708 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA);
9709 if (!TLI->isSDNodeAlwaysUniform(Node))
9710 Node->SDNodeBits.IsDivergent = IsDivergent;
9711 checkForCycles(Node);
9712}
9713
9714SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
9715 SmallVectorImpl<SDValue> &Vals) {
9716 size_t Limit = SDNode::getMaxNumOperands();
9717 while (Vals.size() > Limit) {
9718 unsigned SliceIdx = Vals.size() - Limit;
9719 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
9720 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
9721 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
9722 Vals.emplace_back(NewTF);
9723 }
9724 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
9725}
9726
9727#ifndef NDEBUG
9728static void checkForCyclesHelper(const SDNode *N,
9729 SmallPtrSetImpl<const SDNode*> &Visited,
9730 SmallPtrSetImpl<const SDNode*> &Checked,
9731 const llvm::SelectionDAG *DAG) {
9732 // If this node has already been checked, don't check it again.
9733 if (Checked.count(N))
9734 return;
9735
9736 // If a node has already been visited on this depth-first walk, reject it as
9737 // a cycle.
9738 if (!Visited.insert(N).second) {
9739 errs() << "Detected cycle in SelectionDAG\n";
9740 dbgs() << "Offending node:\n";
9741 N->dumprFull(DAG); dbgs() << "\n";
9742 abort();
9743 }
9744
9745 for (const SDValue &Op : N->op_values())
9746 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
9747
9748 Checked.insert(N);
9749 Visited.erase(N);
9750}
9751#endif
9752
9753void llvm::checkForCycles(const llvm::SDNode *N,
9754 const llvm::SelectionDAG *DAG,
9755 bool force) {
9756#ifndef NDEBUG
9757 bool check = force;
9758#ifdef EXPENSIVE_CHECKS
9759 check = true;
9760#endif // EXPENSIVE_CHECKS
9761 if (check) {
9762 assert(N && "Checking nonexistent SDNode");
9763 SmallPtrSet<const SDNode*, 32> visited;
9764 SmallPtrSet<const SDNode*, 32> checked;
9765 checkForCyclesHelper(N, visited, checked, DAG);
9766 }
9767#endif // !NDEBUG
9768}
9769
9770void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
9771 checkForCycles(DAG->getRoot().getNode(), DAG, force);
9772}
9773